feat: rebuild AudioManager around downloadable moods
CI / chart (push) Successful in 3s
CI / test (push) Successful in 2m2s
CI / chart-push (push) Successful in 4s
CI / image (push) Successful in 3m20s

- V8 wipe migration and the shared song store keyed by YouTube video id
- media store, indexer, downloader, recommendation top-up and refcounted GC
- mood songs plus the recommendation lifecycle and mixed radio pools
- new search, song and recommendation API; remove library/longform/seed routes
- chart PVC/media env, docs, Nix, README and an end-to-end smoke test
This commit is contained in:
TapTap committed 2026-11-15 21:37:39 +00:00
1 parent 50d8b6c580
commit 824eb36dd2
59 files changed
+3838 -5276

No files matched your search

@@ -6,35 +6,28 @@ import java.nio.file.Path
import java.time.Instant
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.update
import org.jetbrains.exposed.sql.transactions.transaction
import org.junit.jupiter.api.io.TempDir
import win.taptap.musicmanager.TestAudio
import win.taptap.musicmanager.config.AppConfig
import win.taptap.musicmanager.config.FolderType
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.MusicFolder
import win.taptap.musicmanager.config.UserAccount
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.AlbumArtistsTable
import win.taptap.musicmanager.db.AlbumsTable
import win.taptap.musicmanager.db.ArtistsTable
import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.DatabaseHandle
import win.taptap.musicmanager.db.DownloadJobsTable
import win.taptap.musicmanager.db.DownloadRepository
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.MoodRecord
import win.taptap.musicmanager.db.RecommendationUpsert
import win.taptap.musicmanager.db.SongArtistsTable
import win.taptap.musicmanager.db.SongGenresTable
import win.taptap.musicmanager.db.SongsTable
import win.taptap.musicmanager.util.IdGenerator
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.db.SongUpsert
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.scanner.CoverArtService
import win.taptap.musicmanager.scanner.Mp3TagExtractor
import win.taptap.musicmanager.scanner.SongIndexer
class DownloadServiceTest {
@@ -42,377 +35,297 @@ class DownloadServiceTest {
lateinit var tempDir: Path
@Test
fun `downloaded tracks are promoted to recommendation matches and seeds`() {
withDatabase { handle, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Drei ???",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
MoodRepository.storeRecommendations(
mood.id,
listOf(
RecommendationUpsert(
videoId = "v-1",
title = "Neue Folge",
artist = "Die drei ???",
album = null,
duration = 300,
thumbnail = null,
matchedSongId = null,
),
),
"2026-01-01T00:00:00Z",
)
DownloadRepository.enqueue("alice", mood.id, "v-1", "Neue Folge", "Die drei ???")
var scanned = false
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { videoId, _, _, dir ->
val file = dir.resolve("$videoId.mp3")
Files.write(file, "fake-mp3".toByteArray())
file
},
tempDir = tempDir.resolve("downloads"),
startScan = {
if (!scanned) {
scanned = true
insertSong(handle, "Die drei ???", "Neue Folge")
}
true
},
isScanning = { false },
)
fun `a manual download lands in the media store and attaches to the mood`() {
TestAudio.assumeFfmpeg()
withService { service, mediaStore, _ ->
val mood = mood(recommendationCount = 0)
DownloadRepository.enqueue("vid-manual", mood.id, DownloadRepository.KIND_SONG)
assertTrue(service.processOne())
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-1"))
val job = assertNotNull(
DownloadRepository.find(mood.id, "vid-manual", DownloadRepository.KIND_SONG),
)
assertEquals("done", job.state)
val songId = assertNotNull(job.songId)
assertEquals(listOf(songId), MoodRepository.songIds(mood.id))
val recommendation = assertNotNull(MoodRepository.recommendation(mood.id, "v-1"))
assertEquals("accepted", recommendation.state)
assertEquals(songId, recommendation.matchedSongId)
assertTrue(MoodRepository.seedSongIds(mood.id).contains(songId))
val imports = library.folders.first().path.resolve("AudioManager Imports")
val files = Files.walk(imports).use { stream ->
stream.filter { Files.isRegularFile(it) }.toList()
}
assertEquals(1, files.size)
assertEquals("Neue Folge.mp3", files.first().fileName.toString())
assertEquals("Neue Folge", files.first().parent.fileName.toString())
assertEquals("Die drei ___", files.first().parent.parent.fileName.toString())
val song = assertNotNull(SongRepository.byId(songId))
assertEquals("vid-manual", song.videoId)
assertEquals("Song vid-manual", song.title)
assertEquals("Artist", song.artist)
assertEquals("vid-manual/Song vid-manual.mp3", song.relativePath)
assertTrue(Files.isRegularFile(assertNotNull(mediaStore.file(song))))
}
}
@Test
fun `an already downloaded song attaches without another download`() {
withService(downloader = AudioDownloader { _, _ -> error("downloader must not run") }) { service, mediaStore, _ ->
val mood = mood(recommendationCount = 0)
val song = insertSong(mediaStore, "vid-existing")
DownloadRepository.enqueue("vid-existing", mood.id, DownloadRepository.KIND_SONG)
assertTrue(service.processOne())
assertEquals(listOf(song.id), MoodRepository.songIds(mood.id))
}
}
@Test
fun `downloaded file is promoted instead of an older same-title song`() {
withDatabase { handle, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Drei ???",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
MoodRepository.storeRecommendations(
fun `the same video is downloaded once for two moods`() {
TestAudio.assumeFfmpeg()
var downloads = 0
withService(downloader = AudioDownloader { videoId, dir -> downloads++; mp3(videoId, dir) }) { service, _, _ ->
val first = mood(recommendationCount = 0, name = "First")
val second = mood(recommendationCount = 0, name = "Second")
DownloadRepository.enqueue("vid-shared", first.id, DownloadRepository.KIND_SONG)
DownloadRepository.enqueue("vid-shared", second.id, DownloadRepository.KIND_SONG)
assertTrue(service.processOne())
assertTrue(service.processOne())
assertEquals(1, downloads)
assertEquals(1, SongRepository.count())
assertEquals(MoodRepository.songIds(first.id), MoodRepository.songIds(second.id))
}
}
@Test
fun `a recommendation download becomes ready but is not a mood song`() {
TestAudio.assumeFfmpeg()
withService { service, _, _ ->
val mood = mood(recommendationCount = 1)
storeRecommendation(mood.id, "vid-rec")
val promoted = MoodRepository.promoteCandidates(mood.id, 1).single()
DownloadRepository.enqueue(
promoted.videoId,
mood.id,
listOf(
RecommendationUpsert(
videoId = "v-dup",
title = "Neue Folge",
artist = "Die drei ???",
album = null,
duration = 300,
thumbnail = null,
matchedSongId = null,
),
),
"2026-01-01T00:00:00Z",
)
DownloadRepository.enqueue("alice", mood.id, "v-dup", "Neue Folge", "Die drei ???")
val oldId = insertSong(
handle,
"Die drei ???",
"Neue Folge",
"Old/Die drei ??? - Neue Folge.mp3",
)
var newId: String? = null
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { videoId, _, _, dir ->
val file = dir.resolve("$videoId.mp3")
Files.write(file, "fake-mp3".toByteArray())
file
},
tempDir = tempDir.resolve("downloads"),
startScan = {
if (newId == null) {
newId = insertSong(
handle,
"Die drei ???",
"Neue Folge",
"AudioManager Imports/Die drei ___/Neue Folge/Neue Folge.mp3",
)
}
true
},
isScanning = { false },
DownloadRepository.KIND_RECOMMENDATION,
)
assertTrue(service.processOne())
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-dup"))
assertEquals("done", job.state)
assertEquals(newId, job.songId)
assertNotEquals(oldId, job.songId)
val recommendation = assertNotNull(
MoodRepository.recommendation(mood.id, "vid-rec"),
)
assertEquals("ready", recommendation.state)
assertNotNull(recommendation.songId)
assertTrue(MoodRepository.songIds(mood.id).isEmpty())
}
}
@Test
fun `failed downloads retry and then fail`() {
withDatabase { _, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Broken",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
fun `adding while downloading attaches when the job finishes`() {
TestAudio.assumeFfmpeg()
withService { service, _, _ ->
val mood = mood(recommendationCount = 1)
storeRecommendation(mood.id, "vid-rec")
MoodRepository.promoteCandidates(mood.id, 1)
DownloadRepository.enqueue(
"vid-rec",
mood.id,
DownloadRepository.KIND_RECOMMENDATION,
)
DownloadRepository.enqueue("alice", mood.id, "v-2", "Missing", "Nobody")
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { _, _, _, _ -> error("network down") },
tempDir = tempDir.resolve("downloads"),
startScan = { true },
isScanning = { false },
MoodRepository.markAdded(mood.id, "vid-rec")
assertTrue(service.processOne())
val recommendation = assertNotNull(MoodRepository.recommendation(mood.id, "vid-rec"))
assertEquals("added", recommendation.state)
assertEquals(listOf(assertNotNull(recommendation.songId)), MoodRepository.songIds(mood.id))
}
}
@Test
fun `rejecting while downloading leaves no mood song and collects the file`() {
TestAudio.assumeFfmpeg()
withService { service, mediaStore, _ ->
val mood = mood(recommendationCount = 1)
storeRecommendation(mood.id, "vid-rec")
MoodRepository.promoteCandidates(mood.id, 1)
DownloadRepository.enqueue(
"vid-rec",
mood.id,
DownloadRepository.KIND_RECOMMENDATION,
)
MoodRepository.markRejected(mood.id, "vid-rec")
assertTrue(service.processOne())
assertTrue(MoodRepository.songIds(mood.id).isEmpty())
assertNull(SongRepository.byVideoId("vid-rec"))
assertTrue(Files.notExists(mediaStore.root.resolve("vid-rec")))
}
}
@Test
fun `failures retry three times and leave the recommendation failed`() {
withService(downloader = AudioDownloader { _, _ -> error("network down") }) { service, _, _ ->
val mood = mood(recommendationCount = 1)
storeRecommendation(mood.id, "vid-broken")
MoodRepository.promoteCandidates(mood.id, 1)
DownloadRepository.enqueue(
"vid-broken",
mood.id,
DownloadRepository.KIND_RECOMMENDATION,
)
assertTrue(service.processOne())
assertEquals("queued", DownloadRepository.forRecommendation("alice", mood.id, "v-2")?.state)
assertTrue(service.processOne())
assertEquals("queued", DownloadRepository.forRecommendation("alice", mood.id, "v-2")?.state)
assertTrue(service.processOne())
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-2"))
val job = assertNotNull(
DownloadRepository.find(mood.id, "vid-broken", DownloadRepository.KIND_RECOMMENDATION),
)
assertEquals("failed", job.state)
assertEquals(3, job.attempts)
assertTrue(job.error?.contains("network down") == true)
}
}
@Test
fun `stale downloading jobs are recovered`() {
withDatabase { handle, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Recovery",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
DownloadRepository.enqueue("alice", mood.id, "v-stale", "Stale", "Nobody")
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-stale"))
transaction(handle.database) {
DownloadJobsTable.update({ DownloadJobsTable.id eq job.id }) {
it[state] = "downloading"
it[updatedAt] = Instant.now().minusSeconds(3600).toString()
}
}
val claimed = assertNotNull(DownloadRepository.claimNext())
assertEquals(job.id, claimed.id)
assertEquals("downloading", claimed.state)
}
}
@Test
fun `failed verification removes the promoted file instead of duplicating it`() {
withDatabase { _, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Broken match",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
DownloadRepository.enqueue("alice", mood.id, "v-nomatch", "No Match", "Nobody")
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { videoId, _, _, dir ->
val file = dir.resolve("$videoId.mp3")
Files.write(file, "fake-mp3".toByteArray())
file
},
tempDir = tempDir.resolve("downloads"),
startScan = { true },
isScanning = { false },
)
repeat(3) { assertTrue(service.processOne()) }
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-nomatch"))
assertEquals("failed", job.state)
val imports = library.folders.first().path.resolve("AudioManager Imports")
if (Files.isDirectory(imports)) {
val files = Files.walk(imports).use { stream ->
stream.filter { Files.isRegularFile(it) }.toList()
}
assertTrue(files.isEmpty(), "leftover promoted files: $files")
}
val recommendation = assertNotNull(MoodRepository.recommendation(mood.id, "vid-broken"))
assertEquals("failed", recommendation.state)
assertTrue(recommendation.error?.contains("network down") == true)
}
}
@Test
fun `permission errors fail without burning retries`() {
withDatabase { _, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Denied",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
DownloadRepository.enqueue("alice", mood.id, "v-3", "Blocked", "Nobody")
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { _, _, _, _ ->
throw AccessDeniedException("/mnt/media/music/AudioManager Imports")
},
tempDir = tempDir.resolve("downloads"),
startScan = { true },
isScanning = { false },
)
withService(
downloader = AudioDownloader { _, _ -> throw AccessDeniedException("/data/media") },
) { service, _, _ ->
val mood = mood(recommendationCount = 0)
DownloadRepository.enqueue("vid-denied", mood.id, DownloadRepository.KIND_SONG)
assertTrue(service.processOne())
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-3"))
val job = assertNotNull(
DownloadRepository.find(mood.id, "vid-denied", DownloadRepository.KIND_SONG),
)
assertEquals("failed", job.state)
assertEquals(1, job.attempts)
assertTrue(job.error?.contains("AccessDeniedException") == true)
}
}
private fun withDatabase(block: (DatabaseHandle, LibraryConfig, UsersConfig) -> Unit) {
val musicDir = Files.createDirectories(tempDir.resolve("music"))
val library = LibraryConfig(
listOf(MusicFolder(1, "Music", FolderType.MUSIC, musicDir)),
@Test
fun `manual adds outrank recommendation downloads`() {
withDatabase { _, _ ->
val mood = mood(recommendationCount = 1)
DownloadRepository.enqueue(
"vid-rec",
mood.id,
DownloadRepository.KIND_RECOMMENDATION,
)
DownloadRepository.enqueue("vid-manual", mood.id, DownloadRepository.KIND_SONG)
val claimed = assertNotNull(DownloadRepository.claimNext())
assertEquals("vid-manual", claimed.videoId)
assertEquals(DownloadRepository.KIND_SONG, claimed.kind)
}
}
private fun mood(recommendationCount: Int, name: String = "Focus"): MoodRecord =
MoodRepository.create(
owner = "alice",
name = name,
description = null,
query = null,
genres = emptyList(),
collectionPercent = 50,
recommendationCount = recommendationCount,
refreshHours = 24,
)
val users = UsersConfig(
listOf(UserAccount("alice", "pw", admin = true, folders = setOf(1))),
)
val handle = DatabaseFactory.connect(
AppConfig(
port = 8080,
dataDir = tempDir.resolve("data"),
libraryConfigPath = tempDir.resolve("library.yaml"),
usersFile = tempDir.resolve("users.yaml"),
ffmpegPath = "ffmpeg",
downloadDir = tempDir.resolve("downloads"),
private fun storeRecommendation(moodId: String, videoId: String) {
MoodRepository.storeCandidates(
moodId,
listOf(
RecommendationUpsert(
videoId = videoId,
title = "Recommended",
artist = "Artist",
album = null,
duration = 180,
thumbnail = null,
),
),
Instant.now().toString(),
)
}
private fun withService(
downloader: AudioDownloader = AudioDownloader { videoId, dir -> mp3(videoId, dir) },
block: (DownloadService, MediaStore, MediaGcService) -> Unit,
) {
withDatabase { handle, store ->
val coverArt = CoverArtService(tempDir.resolve("data"))
val indexer = SongIndexer(store, coverArt)
val gc = MediaGcService(store, coverArt)
val service = DownloadService(
downloader = downloader,
mediaStore = store,
indexer = indexer,
gc = gc,
tagExtractor = Mp3TagExtractor(),
tempDir = tempDir.resolve("downloads"),
)
try {
block(service, store, gc)
} catch (t: Throwable) {
throw t
}
}
}
private fun withDatabase(block: (DatabaseHandle, MediaStore) -> Unit) {
val handle = DatabaseFactory.connect(appConfig())
try {
ConfigRepository.sync(library, users)
block(handle, library, users)
ConfigRepository.sync(UsersConfig(listOf(UserAccount("alice", "pw", admin = true))))
val store = MediaStore(Files.createDirectories(tempDir.resolve("media")))
block(handle, store)
} finally {
handle.close()
}
}
private fun insertSong(
handle: DatabaseHandle,
artistName: String,
title: String,
relativePath: String = "AudioManager Imports/$artistName - $title.mp3",
): String {
val artistId = IdGenerator.artistId(artistName)
val albumId = IdGenerator.albumId(artistName, "Downloads")
val songId = IdGenerator.songId(1, relativePath)
val now = Instant.now().toString()
transaction(handle.database) {
ArtistsTable.insertIgnore {
it[id] = artistId
it[name] = artistName
it[sortName] = artistName
it[searchText] = IdGenerator.normalize(artistName)
}
AlbumsTable.insertIgnore {
it[id] = albumId
it[name] = "Downloads"
it[createdAt] = now
it[searchText] = IdGenerator.normalize("$title $artistName")
}
AlbumArtistsTable.insertIgnore {
it[this.albumId] = albumId
it[this.artistId] = artistId
it[position] = 0
}
SongsTable.insert {
it[id] = songId
it[this.albumId] = albumId
it[folderId] = 1
it[this.title] = title
it[displayArtist] = artistName
it[albumName] = "Downloads"
it[duration] = 300
it[bitRate] = 128
it[size] = 10
it[suffix] = "mp3"
it[contentType] = "audio/mpeg"
it[this.relativePath] = relativePath
it[fileMtime] = 0
it[createdAt] = now
it[updatedAt] = now
it[searchText] = IdGenerator.normalize("$title $artistName Downloads $artistName")
}
SongArtistsTable.insert {
it[this.songId] = songId
it[this.artistId] = artistId
it[position] = 0
}
SongGenresTable.insert {
it[this.songId] = songId
it[genre] = "Hörspiel"
}
}
return songId
private fun mp3(videoId: String, dir: Path): Path {
val file = dir.resolve("$videoId.mp3")
TestAudio.createMp3(
file,
metadata = mapOf(
"title" to "Song $videoId",
"artist" to "Artist",
"album" to "Album",
),
)
return file
}
private fun insertSong(mediaStore: MediaStore, videoId: String): win.taptap.musicmanager.db.SongRecord {
val file = mediaStore.target(videoId, "Existing")
Files.createDirectories(file.parent)
Files.write(file, "fake-mp3".toByteArray())
return SongRepository.upsert(
SongUpsert(
videoId = videoId,
title = "Existing",
artist = "Artist",
album = null,
duration = 180,
bitRate = 128,
size = Files.size(file),
relativePath = mediaStore.root.relativize(file).joinToString("/") { it.toString() },
fileMtime = Files.getLastModifiedTime(file).toMillis(),
thumbnail = null,
),
)
}
private fun appConfig(): AppConfig = AppConfig(
port = 8080,
dataDir = tempDir.resolve("data"),
usersFile = tempDir.resolve("users.yaml"),
ffmpegPath = "ffmpeg",
downloadDir = tempDir.resolve("downloads"),
)
}