fix: harden scans, downloads, sync and radio edge cases
Scanner and library: - Keep the last library when a configured folder cannot be walked; the previous code deleted every song, cascading away mood seeds and progress. - Re-adding a file now clears its tombstone, and a re-deletion refreshes the timestamp instead of keeping the old one. - Tombstone timestamps are unique per row, and delta pages continue from the older of the song/tombstone page ends. - Snapshot pagination accepts a stable revision for later pages so songs inserted mid-sync are not skipped forever. - Reject "%" and "_" searches that normalize to an empty LIKE needle. Downloads and discovery: - Download promotion uses the deterministic path id, one folder per track so covers do not overwrite each other, removes failed promotions, and restarts the scan after a running one finishes. - Stale downloading jobs are recovered; reject cancels queued/in-flight jobs; markDone only owns downloading rows; rejecting no longer seeds the track. - Reuse an already completed download of the same video for another mood. - yt-dlp stdout and stderr are separated, empty discovery results no longer prune undecided recommendations, and filenames are capped by UTF-8 bytes. API and data: - Malformed or missing JSON returns 400 instead of 500; optional session bodies use receiveNullable. - Playback events must belong to the session, known song and open session. - Radio refills the candidate pool after recent-song pruning. - Seed percent is required in MoodRepository.update. - Normalize legacy V4 liked/downloaded recommendation states before V5, with an upgrade regression test. CI: check out without persisting credentials and require tests before chart publishing.
This commit is contained in:
1 parent
c75f2971b4
commit
4be502bb4e
23 files changed
+705
-66
No files matched your search
@@ -3,6 +3,8 @@ package win.taptap.musicmanager.api
|
||||
import io.ktor.http.HttpHeaders
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import io.ktor.server.application.ApplicationCall
|
||||
import io.ktor.server.plugins.BadRequestException
|
||||
import io.ktor.server.plugins.ContentTransformationException
|
||||
import io.ktor.server.request.receive
|
||||
import io.ktor.server.response.respond
|
||||
import io.ktor.util.AttributeKey
|
||||
@@ -94,6 +96,10 @@ suspend inline fun <reified T : Any> ApplicationCall.receiveJson(): T =
|
||||
receive<T>()
|
||||
} catch (e: SerializationException) {
|
||||
throw ApiException(HttpStatusCode.BadRequest, "invalid_body", "Invalid JSON body")
|
||||
} catch (e: ContentTransformationException) {
|
||||
throw ApiException(HttpStatusCode.BadRequest, "invalid_body", "Invalid JSON body")
|
||||
} catch (e: BadRequestException) {
|
||||
throw ApiException(HttpStatusCode.BadRequest, "invalid_body", "Invalid JSON body")
|
||||
} catch (e: IllegalArgumentException) {
|
||||
throw ApiException(HttpStatusCode.BadRequest, "invalid_body", e.message ?: "Invalid body")
|
||||
}
|
||||
|
||||
@@ -43,17 +43,35 @@ fun Route.libraryRoutes(
|
||||
val now = Instant.now().toString()
|
||||
val since = call.request.queryParameters["since"]?.takeIf { it.isNotBlank() }
|
||||
val cursor = call.request.queryParameters["cursor"]?.takeIf { it.isNotBlank() }
|
||||
// Snapshot pagination must keep the timestamp of the first page. Using
|
||||
// each page's own time could hide a song inserted between pages, whose
|
||||
// id sorts before the cursor but whose updatedAt predates the final
|
||||
// revision (so no later delta would fetch it).
|
||||
val revision = call.request.queryParameters["revision"]?.takeIf { it.isNotBlank() } ?: now
|
||||
|
||||
if (since != null) {
|
||||
val changed = LibraryRepository.changedSongs(since, folders, limit, user.name)
|
||||
val deleted = LibraryRepository.tombstones(since, limit).map { it.entityId }
|
||||
val complete = changed.size < limit && deleted.size < limit
|
||||
val deletedRows = LibraryRepository.tombstones(since, limit)
|
||||
val deleted = deletedRows.map { it.entityId }
|
||||
val complete = changed.size < limit && deletedRows.size < limit
|
||||
|
||||
// Either list can fill a page. Continue from the older of the two
|
||||
// page ends, so a full tombstone page is not lost when fewer songs
|
||||
// changed. Re-reading a few rows is harmless; skipping them is not.
|
||||
val nextSince = when {
|
||||
changed.size == limit && deletedRows.size == limit ->
|
||||
listOf(changed.last().updatedAt, deletedRows.last().deletedAt).minOrNull()
|
||||
changed.size == limit -> changed.last().updatedAt
|
||||
deletedRows.size == limit -> deletedRows.last().deletedAt
|
||||
else -> null
|
||||
}
|
||||
|
||||
call.respond(
|
||||
LibraryPage(
|
||||
songs = changed.map { it.toDto() },
|
||||
deletedSongIds = deleted,
|
||||
revision = if (complete) now else changed.lastOrNull()?.updatedAt ?: now,
|
||||
nextSince = if (changed.size == limit) changed.last().updatedAt else null,
|
||||
revision = if (complete) now else nextSince ?: now,
|
||||
nextSince = nextSince,
|
||||
),
|
||||
)
|
||||
return@get
|
||||
@@ -63,7 +81,7 @@ fun Route.libraryRoutes(
|
||||
call.respond(
|
||||
LibraryPage(
|
||||
songs = songs.map { it.toDto() },
|
||||
revision = now,
|
||||
revision = revision,
|
||||
nextCursor = if (songs.size == limit) songs.last().id else null,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -183,6 +183,9 @@ fun Route.moodRoutes(
|
||||
|
||||
val matched = recommendation.matchedSongId?.let { songId ->
|
||||
LibraryRepository.song(songId, user.musicFolderIds(library), user.name)
|
||||
} ?: DownloadRepository.completedSongForVideo(recommendation.videoId)?.let { songId ->
|
||||
// The same video may already have been downloaded for another mood.
|
||||
LibraryRepository.song(songId, user.musicFolderIds(library), user.name)
|
||||
}
|
||||
val seedAdded = matched != null &&
|
||||
MoodRepository.addSeed(mood.id, user.name, matched.id, "recommendation")
|
||||
@@ -208,6 +211,7 @@ fun Route.moodRoutes(
|
||||
val videoId = call.parameters["videoId"] ?: badRequest("Video id is required")
|
||||
MoodRepository.recommendation(mood.id, videoId) ?: notFound("Recommendation")
|
||||
MoodRepository.setRecommendationState(mood.id, videoId, "rejected")
|
||||
DownloadRepository.cancel(mood.id, videoId, "Recommendation rejected")
|
||||
call.respond(decision(mood.id, videoId, user, library, seedAdded = false))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package win.taptap.musicmanager.api
|
||||
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import io.ktor.server.request.receiveNullable
|
||||
import io.ktor.server.response.respond
|
||||
import io.ktor.server.routing.Route
|
||||
import io.ktor.server.routing.delete
|
||||
@@ -37,8 +38,7 @@ fun Route.sessionRoutes(
|
||||
post("/moods/{id}/sessions") {
|
||||
val user = call.requireApiUser(users)
|
||||
val mood = ownedMoodFor(call.parameters["id"], user)
|
||||
val request = runCatching { call.receiveJson<StartSessionRequest>() }
|
||||
.getOrDefault(StartSessionRequest())
|
||||
val request = call.receiveNullable<StartSessionRequest>() ?: StartSessionRequest()
|
||||
val limit = request.limit ?: DEFAULT_BATCH
|
||||
|
||||
val started = radioService.start(mood, user, library, limit)
|
||||
@@ -81,8 +81,7 @@ fun Route.sessionRoutes(
|
||||
val session = ownedSession(call.parameters["id"], user)
|
||||
if (session.closedAt != null) conflict("Session is closed")
|
||||
val mood = ownedMoodFor(session.moodId, user)
|
||||
val request = runCatching { call.receiveJson<NextTracksRequest>() }
|
||||
.getOrDefault(NextTracksRequest())
|
||||
val request = call.receiveNullable<NextTracksRequest>() ?: NextTracksRequest()
|
||||
|
||||
val songs = radioService.next(
|
||||
mood = mood,
|
||||
|
||||
@@ -62,12 +62,33 @@ object DatabaseFactory {
|
||||
}
|
||||
}
|
||||
|
||||
normalizeLegacyMoodStates(dataSource)
|
||||
val database = Database.connect(dataSource)
|
||||
migrate(dataSource)
|
||||
|
||||
return DatabaseHandle(dataSource, database)
|
||||
}
|
||||
|
||||
/**
|
||||
* V4 allowed `liked` and `downloaded` recommendations, but its CHECK
|
||||
* constraint rejects the V5 `accepted` state, so an in-place rename is not
|
||||
* possible. Move those rows back to `new` (allowed by both schemas) before
|
||||
* Flyway runs. The table is absent on a fresh database and already uses the
|
||||
* new states after V5, so both are harmless no-ops.
|
||||
*/
|
||||
private fun normalizeLegacyMoodStates(dataSource: HikariDataSource) {
|
||||
dataSource.connection.use { connection ->
|
||||
runCatching {
|
||||
connection.createStatement().use { statement ->
|
||||
statement.executeUpdate(
|
||||
"UPDATE mood_recommendations " +
|
||||
"SET state = 'new' WHERE state IN ('liked', 'downloaded')",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun migrate(dataSource: HikariDataSource) {
|
||||
Flyway.configure()
|
||||
.dataSource(dataSource)
|
||||
@@ -75,4 +96,5 @@ object DatabaseFactory {
|
||||
.load()
|
||||
.migrate()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package win.taptap.musicmanager.db
|
||||
import java.time.Instant
|
||||
import org.jetbrains.exposed.sql.SortOrder
|
||||
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
|
||||
import org.jetbrains.exposed.sql.SqlExpressionBuilder.less
|
||||
import org.jetbrains.exposed.sql.and
|
||||
import org.jetbrains.exposed.sql.insertIgnore
|
||||
import org.jetbrains.exposed.sql.selectAll
|
||||
@@ -73,6 +74,7 @@ object DownloadRepository {
|
||||
|
||||
/** Takes the oldest queued job and marks it as downloading. */
|
||||
fun claimNext(): DownloadJobRecord? = transaction {
|
||||
recoverStale()
|
||||
val row = DownloadJobsTable.selectAll()
|
||||
.where { DownloadJobsTable.state eq "queued" }
|
||||
.orderBy(DownloadJobsTable.createdAt to SortOrder.ASC)
|
||||
@@ -91,7 +93,12 @@ object DownloadRepository {
|
||||
|
||||
fun markDone(id: String, songId: String) {
|
||||
transaction {
|
||||
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
|
||||
// Only finish a job that is still ours. A reject or recovery may
|
||||
// have changed the row while the downloader was running.
|
||||
DownloadJobsTable.update({
|
||||
(DownloadJobsTable.id eq id) and
|
||||
(DownloadJobsTable.state eq "downloading")
|
||||
}) {
|
||||
it[state] = "done"
|
||||
it[this.songId] = songId
|
||||
it[error] = null
|
||||
@@ -100,6 +107,53 @@ object DownloadRepository {
|
||||
}
|
||||
}
|
||||
|
||||
/** Rejects must stop a queued or in-flight download. */
|
||||
fun cancel(moodId: String, videoId: String, reason: String) {
|
||||
transaction {
|
||||
DownloadJobsTable.update({
|
||||
(DownloadJobsTable.moodId eq moodId) and
|
||||
(DownloadJobsTable.videoId eq videoId) and
|
||||
(DownloadJobsTable.state inList listOf("queued", "downloading"))
|
||||
}) {
|
||||
it[state] = "failed"
|
||||
it[error] = reason.take(1000)
|
||||
it[updatedAt] = Instant.now().toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A pod restart or an uncaught bookkeeping failure can leave jobs in
|
||||
* `downloading`. Requeue them after a generous timeout so they are not lost.
|
||||
*/
|
||||
private fun recoverStale(now: Instant = Instant.now()) {
|
||||
val cutoff = now.minusSeconds(STALE_DOWNLOAD_SECONDS).toString()
|
||||
val stale = DownloadJobsTable.selectAll()
|
||||
.where {
|
||||
(DownloadJobsTable.state eq "downloading") and
|
||||
(DownloadJobsTable.updatedAt less cutoff)
|
||||
}
|
||||
.toList()
|
||||
|
||||
stale.forEach { row ->
|
||||
val id = row[DownloadJobsTable.id]
|
||||
val attempts = row[DownloadJobsTable.attempts]
|
||||
if (attempts >= MAX_ATTEMPTS) {
|
||||
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
|
||||
it[state] = "failed"
|
||||
it[error] = "Download was interrupted and could not be resumed"
|
||||
it[updatedAt] = now.toString()
|
||||
}
|
||||
} else {
|
||||
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
|
||||
it[state] = "queued"
|
||||
it[error] = "Download was interrupted; retrying"
|
||||
it[updatedAt] = now.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun markFailed(id: String, error: String) {
|
||||
transaction {
|
||||
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
|
||||
@@ -136,6 +190,19 @@ object DownloadRepository {
|
||||
.associate { it[DownloadJobsTable.videoId] to it.toRecord() }
|
||||
}
|
||||
|
||||
/** A video already downloaded for any mood, if the current user can see it. */
|
||||
fun completedSongForVideo(videoId: String): String? = transaction {
|
||||
DownloadJobsTable.selectAll()
|
||||
.where {
|
||||
(DownloadJobsTable.videoId eq videoId) and
|
||||
(DownloadJobsTable.state eq "done")
|
||||
}
|
||||
.orderBy(DownloadJobsTable.updatedAt to SortOrder.DESC)
|
||||
.limit(1)
|
||||
.singleOrNull()
|
||||
?.get(DownloadJobsTable.songId)
|
||||
}
|
||||
|
||||
fun forRecommendation(userName: String, moodId: String, videoId: String): DownloadJobRecord? =
|
||||
transaction {
|
||||
DownloadJobsTable.selectAll()
|
||||
@@ -148,6 +215,9 @@ object DownloadRepository {
|
||||
?.toRecord()
|
||||
}
|
||||
|
||||
private const val STALE_DOWNLOAD_SECONDS = 30 * 60L
|
||||
private const val MAX_ATTEMPTS = 3
|
||||
|
||||
private fun org.jetbrains.exposed.sql.ResultRow.toRecord() = DownloadJobRecord(
|
||||
id = this[DownloadJobsTable.id],
|
||||
userName = this[DownloadJobsTable.userName],
|
||||
|
||||
@@ -658,6 +658,9 @@ object LibraryRepository {
|
||||
if (folderIds.isEmpty()) return@read emptyList()
|
||||
|
||||
val pattern = likePattern(query)
|
||||
// "%" or "_" alone normalizes to an empty LIKE needle and would match
|
||||
// the whole library.
|
||||
if (pattern == "%%") return@read emptyList()
|
||||
val rows = SongsTable.selectAll()
|
||||
.where {
|
||||
(SongsTable.searchText like pattern) and
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package win.taptap.musicmanager.db
|
||||
|
||||
import java.time.Instant
|
||||
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
|
||||
import org.jetbrains.exposed.sql.and
|
||||
import org.jetbrains.exposed.sql.deleteWhere
|
||||
import org.jetbrains.exposed.sql.insert
|
||||
@@ -138,6 +139,12 @@ object MediaRepository {
|
||||
}
|
||||
}
|
||||
|
||||
// A re-added song must not stay tombstoned for clients that still
|
||||
// remember the older revision.
|
||||
LibraryTombstonesTable.deleteWhere {
|
||||
it.run { (entity eq "song") and (entityId eq track.songId) }
|
||||
}
|
||||
|
||||
SongArtistsTable.deleteWhere { it.run { SongArtistsTable.songId eq track.songId } }
|
||||
track.artists.forEachIndexed { index, artist ->
|
||||
SongArtistsTable.insertIgnore {
|
||||
@@ -171,12 +178,23 @@ object MediaRepository {
|
||||
.map { it[SongsTable.id] }
|
||||
.filterNot { it in seenSongIds }
|
||||
|
||||
val now = Instant.now().toString()
|
||||
obsolete.forEach { songId ->
|
||||
val base = Instant.now()
|
||||
obsolete.forEachIndexed { index, songId ->
|
||||
// Unique timestamps let timestamp cursors advance even when a
|
||||
// scan deletes hundreds of songs at once.
|
||||
val timestamp = base.plusNanos(index.toLong()).toString()
|
||||
LibraryTombstonesTable.insertIgnore {
|
||||
it[entity] = "song"
|
||||
it[entityId] = songId
|
||||
it[deletedAt] = now
|
||||
it[deletedAt] = timestamp
|
||||
}
|
||||
// insertIgnore keeps the old timestamp on a re-deletion, which
|
||||
// would make the second deletion invisible to synced clients.
|
||||
LibraryTombstonesTable.update({
|
||||
(LibraryTombstonesTable.entity eq "song") and
|
||||
(LibraryTombstonesTable.entityId eq songId)
|
||||
}) {
|
||||
it[deletedAt] = timestamp
|
||||
}
|
||||
SongsTable.deleteWhere { it.run { SongsTable.id eq songId } }
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ object MoodRepository {
|
||||
targetSize: Int,
|
||||
refreshHours: Int,
|
||||
ytEnabled: Boolean,
|
||||
seedPercent: Int = 50,
|
||||
seedPercent: Int,
|
||||
): MoodRecord? = transaction {
|
||||
val exists = MoodsTable.selectAll()
|
||||
.where { (MoodsTable.id eq id) and (MoodsTable.owner eq owner) }
|
||||
@@ -351,16 +351,19 @@ object MoodRepository {
|
||||
}
|
||||
|
||||
/** Links a downloaded recommendation to the scanned song and accepts it. */
|
||||
fun attachDownloadedSong(moodId: String, videoId: String, songId: String) {
|
||||
transaction {
|
||||
val row = MoodRecommendationsTable.selectAll()
|
||||
.where {
|
||||
(MoodRecommendationsTable.moodId eq moodId) and
|
||||
(MoodRecommendationsTable.videoId eq videoId)
|
||||
}
|
||||
.singleOrNull()
|
||||
?: return@transaction
|
||||
MoodRecommendationsTable.update({
|
||||
fun attachDownloadedSong(moodId: String, videoId: String, songId: String): Boolean = transaction {
|
||||
val row = MoodRecommendationsTable.selectAll()
|
||||
.where {
|
||||
(MoodRecommendationsTable.moodId eq moodId) and
|
||||
(MoodRecommendationsTable.videoId eq videoId)
|
||||
}
|
||||
.singleOrNull()
|
||||
?: return@transaction false
|
||||
// The user may have rejected the track while it was downloading.
|
||||
if (row[MoodRecommendationsTable.state] == "rejected") {
|
||||
return@transaction false
|
||||
}
|
||||
MoodRecommendationsTable.update({
|
||||
(MoodRecommendationsTable.moodId eq moodId) and
|
||||
(MoodRecommendationsTable.videoId eq videoId)
|
||||
}) {
|
||||
@@ -368,8 +371,8 @@ object MoodRepository {
|
||||
it[state] = "accepted"
|
||||
it[timesAdded] = row[MoodRecommendationsTable.timesAdded] + 1
|
||||
it[lastSeen] = Instant.now().toString()
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fun markDiscovered(id: String) {
|
||||
|
||||
@@ -144,6 +144,17 @@ object RadioRepository {
|
||||
(PlaybackSessionsTable.userName eq userName)
|
||||
}
|
||||
.singleOrNull() ?: return@transaction false
|
||||
if (session[PlaybackSessionsTable.closedAt] != null) return@transaction false
|
||||
|
||||
// Only tracks the server actually queued in this session may change
|
||||
// affinity. Arbitrary/deleted ids used to violate the affinity FK.
|
||||
val queued = SessionItemsTable.selectAll()
|
||||
.where {
|
||||
(SessionItemsTable.sessionId eq sessionId) and
|
||||
(SessionItemsTable.songId eq songId)
|
||||
}
|
||||
.any()
|
||||
if (!queued) return@transaction false
|
||||
|
||||
val now = Instant.now().toString()
|
||||
SessionItemsTable.update({
|
||||
|
||||
@@ -91,6 +91,12 @@ class DiscoveryService(
|
||||
}
|
||||
}
|
||||
|
||||
if (candidates.isEmpty()) {
|
||||
val message = "YouTube returned no usable results"
|
||||
MoodRepository.markError(mood.id, message)
|
||||
return DiscoveryOutcome(0, 0, error = message)
|
||||
}
|
||||
|
||||
val known = MoodRepository.recommendations(mood.id).associateBy { it.videoId }
|
||||
val moodText = listOfNotNull(mood.query, mood.description, mood.name)
|
||||
.joinToString(" ")
|
||||
@@ -216,5 +222,8 @@ internal fun artistMatches(a: String, b: String): Boolean {
|
||||
val left = matchKey(a)
|
||||
val right = matchKey(b)
|
||||
if (left.isEmpty() || right.isEmpty()) return false
|
||||
return left == right || left.contains(right) || right.contains(left)
|
||||
if (left == right) return true
|
||||
// Short names like "A" used to match every artist containing the letter.
|
||||
if (left.length < 3 || right.length < 3) return false
|
||||
return " $left ".contains(" $right ") || " $right ".contains(" $left ")
|
||||
}
|
||||
@@ -44,24 +44,34 @@ class YtDlpSearch(
|
||||
"10",
|
||||
"ytsearch${limit.coerceIn(1, 25)}:$query",
|
||||
)
|
||||
.redirectErrorStream(true)
|
||||
// Warnings and ERROR lines must not be mixed into stdout: the JSON
|
||||
// parser would fail and look like an empty result set.
|
||||
.redirectErrorStream(false)
|
||||
.start()
|
||||
|
||||
val output = StringBuilder()
|
||||
val reader = thread(isDaemon = true, name = "yt-dlp-search") {
|
||||
val errors = StringBuilder()
|
||||
val reader = thread(isDaemon = true, name = "yt-dlp-search-stdout") {
|
||||
process.inputStream.bufferedReader().useLines { lines ->
|
||||
lines.forEach { output.appendLine(it) }
|
||||
}
|
||||
}
|
||||
val errorReader = thread(isDaemon = true, name = "yt-dlp-search-stderr") {
|
||||
process.errorStream.bufferedReader().useLines { lines ->
|
||||
lines.forEach { errors.appendLine(it) }
|
||||
}
|
||||
}
|
||||
|
||||
if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
|
||||
process.destroyForcibly()
|
||||
reader.join(2_000)
|
||||
errorReader.join(2_000)
|
||||
throw IllegalStateException("yt-dlp timed out after ${timeoutSeconds}s")
|
||||
}
|
||||
reader.join(2_000)
|
||||
errorReader.join(2_000)
|
||||
if (process.exitValue() != 0) {
|
||||
val detail = output.toString().take(400).trim()
|
||||
val detail = (errors.toString().ifBlank { output.toString() }).take(400).trim()
|
||||
throw IllegalStateException("yt-dlp exited with ${process.exitValue()}: $detail")
|
||||
}
|
||||
return parseYtSearchOutput(output.toString())
|
||||
|
||||
@@ -16,6 +16,7 @@ import win.taptap.musicmanager.db.LibraryRepository
|
||||
import win.taptap.musicmanager.db.MoodRepository
|
||||
import win.taptap.musicmanager.discovery.artistMatches
|
||||
import win.taptap.musicmanager.discovery.matchKey
|
||||
import win.taptap.musicmanager.util.IdGenerator
|
||||
|
||||
fun interface AudioDownloader {
|
||||
/** Downloads [videoId] and returns the resulting MP3 file. */
|
||||
@@ -72,10 +73,17 @@ class DownloadService(
|
||||
val working = tempDir.resolve(job.id)
|
||||
Files.createDirectories(working)
|
||||
|
||||
var promoted: Path? = null
|
||||
try {
|
||||
val downloaded = downloader.download(job.videoId, job.title, job.artist, working)
|
||||
val extension = downloaded.fileName.toString().substringAfterLast('.', "mp3")
|
||||
val target = uniqueTarget(importDir, job.artist, job.title, extension)
|
||||
// One folder per artist/title keeps the scanner's album fallback
|
||||
// unique, so two downloads never overwrite each other's cover.
|
||||
val targetDir = importDir
|
||||
.resolve(sanitize(job.artist))
|
||||
.resolve(sanitize(job.title))
|
||||
Files.createDirectories(targetDir)
|
||||
val target = uniqueTarget(targetDir, job.title, extension)
|
||||
try {
|
||||
Files.move(downloaded, target, StandardCopyOption.REPLACE_EXISTING)
|
||||
} catch (e: IOException) {
|
||||
@@ -84,15 +92,33 @@ class DownloadService(
|
||||
e,
|
||||
)
|
||||
}
|
||||
promoted = target
|
||||
log.info("Downloaded {} to {}", job.videoId, target)
|
||||
|
||||
awaitScan()
|
||||
|
||||
val songId = matchSong(job, folders.keys, user.name)
|
||||
// The scanner derives the song id from the folder and relative
|
||||
// path. Prefer that exact id: metadata matching could otherwise
|
||||
// attach the download to an older track with the same title.
|
||||
val relativePath = folder.path.relativize(target).joinToString("/") { it.toString() }
|
||||
val expectedId = IdGenerator.songId(folder.id, relativePath)
|
||||
val songId = LibraryRepository.song(expectedId, folders.keys, user.name)?.id
|
||||
?: matchSong(job, folders.keys, user.name)
|
||||
?: throw IllegalStateException("Downloaded file was not indexed by the scanner")
|
||||
DownloadRepository.markDone(job.id, songId)
|
||||
MoodRepository.attachDownloadedSong(job.moodId, job.videoId, songId)
|
||||
MoodRepository.addSeed(job.moodId, job.userName, songId, "yt")
|
||||
val attached = MoodRepository.attachDownloadedSong(job.moodId, job.videoId, songId)
|
||||
if (attached) {
|
||||
MoodRepository.addSeed(job.moodId, job.userName, songId, "yt")
|
||||
promoted = null
|
||||
} else {
|
||||
log.info("Recommendation {} was rejected; keeping the file but not seeding", job.videoId)
|
||||
promoted = null
|
||||
DownloadRepository.cancel(job.moodId, job.videoId, "Recommendation rejected")
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
// A failed verification must not leave a duplicate for the retry.
|
||||
promoted?.let { target -> runCatching { Files.deleteIfExists(target) } }
|
||||
throw t
|
||||
} finally {
|
||||
runCatching { working.toFile().deleteRecursively() }
|
||||
}
|
||||
@@ -115,9 +141,23 @@ class DownloadService(
|
||||
}
|
||||
|
||||
private fun awaitScan() {
|
||||
// If another scan is already running, wait for that one instead.
|
||||
startScan()
|
||||
// The downloaded file must be covered by a scan that starts *after* it
|
||||
// was moved. If a scan is already running, wait for it and then start a
|
||||
// fresh one; otherwise a slow scan could finish without seeing the file
|
||||
// and the job would download the same track again on retry.
|
||||
val deadline = System.nanoTime() + SCAN_TIMEOUT_MINUTES * 60_000_000_000L
|
||||
var attempts = 0
|
||||
while (!startScan()) {
|
||||
attempts += 1
|
||||
if (attempts >= MAX_SCAN_START_ATTEMPTS) {
|
||||
throw IllegalStateException("Could not start a library scan")
|
||||
}
|
||||
awaitScanToFinish(deadline)
|
||||
}
|
||||
awaitScanToFinish(deadline)
|
||||
}
|
||||
|
||||
private fun awaitScanToFinish(deadline: Long) {
|
||||
while (isScanning()) {
|
||||
if (System.nanoTime() > deadline) {
|
||||
throw IllegalStateException("Library scan did not finish in time")
|
||||
@@ -136,8 +176,8 @@ class DownloadService(
|
||||
?.id
|
||||
}
|
||||
|
||||
private fun uniqueTarget(dir: Path, artist: String, title: String, extension: String): Path {
|
||||
val base = "${sanitize(artist)} - ${sanitize(title)}"
|
||||
private fun uniqueTarget(dir: Path, title: String, extension: String): Path {
|
||||
val base = sanitize(title)
|
||||
val suffix = extension.ifBlank { "mp3" }.take(10)
|
||||
var candidate = dir.resolve("$base.$suffix")
|
||||
var counter = 2
|
||||
@@ -171,14 +211,31 @@ class DownloadService(
|
||||
}
|
||||
|
||||
private fun sanitize(value: String): String =
|
||||
value.replace(Regex("[\\\\/:*?\"<>|\\p{Cntrl}]"), "_")
|
||||
.replace(Regex("\\s+"), " ")
|
||||
.trim()
|
||||
.take(120)
|
||||
.ifEmpty { "track" }
|
||||
truncateUtf8(
|
||||
value.replace(Regex("[\\\\/:*?\"<>|\\p{Cntrl}]"), "_")
|
||||
.replace(Regex("\\s+"), " ")
|
||||
.trim()
|
||||
.ifEmpty { "track" },
|
||||
MAX_NAME_BYTES,
|
||||
)
|
||||
|
||||
/** Linux NAME_MAX is 255 bytes, not characters; CJK titles are 3x. */
|
||||
private fun truncateUtf8(value: String, maxBytes: Int): String {
|
||||
var bytes = 0
|
||||
val result = StringBuilder()
|
||||
for (character in value) {
|
||||
val length = character.toString().toByteArray(Charsets.UTF_8).size
|
||||
if (bytes + length > maxBytes) break
|
||||
result.append(character)
|
||||
bytes += length
|
||||
}
|
||||
return result.toString().ifEmpty { "track" }
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val IMPORT_DIR = "AudioManager Imports"
|
||||
const val SCAN_TIMEOUT_MINUTES = 10L
|
||||
const val MAX_SCAN_START_ATTEMPTS = 5
|
||||
const val MAX_NAME_BYTES = 120
|
||||
}
|
||||
}
|
||||
@@ -36,25 +36,34 @@ class YtDlpDownloader(
|
||||
targetDir.resolve("%(id)s.%(ext)s").toString(),
|
||||
"https://www.youtube.com/watch?v=$videoId",
|
||||
)
|
||||
.redirectErrorStream(true)
|
||||
.redirectErrorStream(false)
|
||||
.start()
|
||||
|
||||
val output = StringBuilder()
|
||||
val reader = thread(isDaemon = true, name = "yt-dlp-download") {
|
||||
val errors = StringBuilder()
|
||||
val reader = thread(isDaemon = true, name = "yt-dlp-download-stdout") {
|
||||
process.inputStream.bufferedReader().useLines { lines ->
|
||||
lines.forEach { output.appendLine(it) }
|
||||
}
|
||||
}
|
||||
val errorReader = thread(isDaemon = true, name = "yt-dlp-download-stderr") {
|
||||
process.errorStream.bufferedReader().useLines { lines ->
|
||||
lines.forEach { errors.appendLine(it) }
|
||||
}
|
||||
}
|
||||
|
||||
if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
|
||||
process.destroyForcibly()
|
||||
reader.join(2_000)
|
||||
errorReader.join(2_000)
|
||||
throw IllegalStateException("yt-dlp download timed out after ${timeoutSeconds}s")
|
||||
}
|
||||
reader.join(2_000)
|
||||
errorReader.join(2_000)
|
||||
if (process.exitValue() != 0) {
|
||||
val detail = (errors.toString().ifBlank { output.toString() }).take(400).trim()
|
||||
throw IllegalStateException(
|
||||
"yt-dlp exited with ${process.exitValue()}: ${output.toString().take(400).trim()}",
|
||||
"yt-dlp exited with ${process.exitValue()}: $detail",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -129,18 +129,7 @@ class RadioService(
|
||||
// Small libraries can exhaust the mood. Replay tracks the session already
|
||||
// heard instead of returning silence, but never repeat the current batch.
|
||||
if (candidates.size < limit) {
|
||||
LibraryRepository.randomSongs(
|
||||
size = max(limit * 10, 200),
|
||||
genre = null,
|
||||
fromYear = null,
|
||||
toYear = null,
|
||||
folderIds = folders,
|
||||
userName = user.name,
|
||||
).forEach { song ->
|
||||
if (song.id !in excluded) {
|
||||
candidates.putIfAbsent(song.id, song)
|
||||
}
|
||||
}
|
||||
refill(candidates, folders, user.name, excluded, sessionSongs, emptySet(), limit)
|
||||
}
|
||||
|
||||
val recent = RadioRepository.recentSongIds(
|
||||
@@ -152,6 +141,14 @@ class RadioService(
|
||||
if (candidates.size > limit * 3) {
|
||||
candidates.keys.removeAll(recent - seedIds)
|
||||
}
|
||||
// Pruning can empty the pool even though the library has songs. Prefer
|
||||
// non-recent tracks, then fall back to replay if the library is tiny.
|
||||
if (candidates.size < limit) {
|
||||
refill(candidates, folders, user.name, excluded, sessionSongs, recent, limit)
|
||||
if (candidates.size < limit) {
|
||||
refill(candidates, folders, user.name, excluded, emptySet(), emptySet(), limit)
|
||||
}
|
||||
}
|
||||
|
||||
val affinity = RadioRepository.affinity(user.name, candidates.keys)
|
||||
val scored = candidates.values.map { song ->
|
||||
@@ -181,6 +178,29 @@ class RadioService(
|
||||
return picks
|
||||
}
|
||||
|
||||
private fun refill(
|
||||
candidates: LinkedHashMap<String, SongRecord>,
|
||||
folders: Set<Int>,
|
||||
userName: String,
|
||||
excluded: Set<String>,
|
||||
avoidSession: Set<String>,
|
||||
avoidRecent: Set<String>,
|
||||
limit: Int,
|
||||
) {
|
||||
LibraryRepository.randomSongs(
|
||||
size = max(limit * 10, 200),
|
||||
genre = null,
|
||||
fromYear = null,
|
||||
toYear = null,
|
||||
folderIds = folders,
|
||||
userName = userName,
|
||||
).forEach { song ->
|
||||
if (song.id !in excluded && song.id !in avoidSession && song.id !in avoidRecent) {
|
||||
candidates.putIfAbsent(song.id, song)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal fun scoreCandidate(
|
||||
song: SongRecord,
|
||||
seedIds: Set<String>,
|
||||
|
||||
@@ -49,6 +49,7 @@ class MediaScanner(
|
||||
var count = 0L
|
||||
val seenSongIds = mutableSetOf<String>()
|
||||
var error: String? = null
|
||||
var folderFailed = false
|
||||
|
||||
try {
|
||||
library.folders.forEach { folder ->
|
||||
@@ -58,11 +59,22 @@ class MediaScanner(
|
||||
status.set(ScanStatus(scanning = true, count = count))
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
folderFailed = true
|
||||
if (error == null) {
|
||||
error = "Could not scan folder ${folder.id} (${folder.path}): " +
|
||||
(e.message ?: e::class.simpleName)
|
||||
}
|
||||
log.error("Could not scan folder ${folder.id} (${folder.path})", e)
|
||||
}
|
||||
}
|
||||
|
||||
repository.deleteSongsNotIn(seenSongIds)
|
||||
if (folderFailed) {
|
||||
// An unmounted or temporarily unreadable folder must not be
|
||||
// mistaken for deleted files. Keep the last known library.
|
||||
log.warn("Skipping orphan cleanup because at least one folder failed to scan")
|
||||
} else {
|
||||
repository.deleteSongsNotIn(seenSongIds)
|
||||
}
|
||||
repository.rebuildAggregates()
|
||||
} catch (t: Throwable) {
|
||||
error = t.message ?: t::class.simpleName
|
||||
|
||||
Reference in new issue
Block a user