feat: accept, store and expose client diagnostics
CI / chart (push) Successful in 3s
CI / test (push) Successful in 2m20s
CI / chart-push (push) Successful in 6s
CI / image (push) Successful in 3m39s

- V11 client_logs table with per-user storage and retention/pruning
- POST/GET/DELETE /logs with batch validation, rate limit and admin access
- redact bearer tokens, Authorization headers and query/JSON secrets twice
- add X-Request-Id to responses and include it in the server log pattern
- report client error entries immediately in the server log
- scripts/logs.sh for source-side retrieval and clearing
- chart knobs CLIENT_LOG_RETENTION_DAYS and CLIENT_LOG_MAX_ROWS_PER_USER
This commit is contained in:
TapTap committed 2026-11-20 13:43:05 +00:00
1 parent 91312faa1c
commit 44b75c4d4d
19 files changed
+981 -5

No files matched your search

@@ -9,6 +9,8 @@ import io.ktor.server.application.install
import io.ktor.server.engine.embeddedServer
import io.ktor.server.netty.Netty
import io.ktor.server.plugins.autohead.AutoHeadResponse
import io.ktor.server.plugins.callid.CallId
import io.ktor.server.plugins.callid.callIdMdc
import io.ktor.server.plugins.calllogging.CallLogging
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
import io.ktor.server.plugins.cors.routing.CORS
@@ -20,6 +22,7 @@ import io.ktor.server.response.respondText
import io.ktor.server.routing.get
import io.ktor.server.routing.routing
import java.nio.file.Files
import java.util.UUID
import java.util.concurrent.TimeUnit
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -38,6 +41,7 @@ import win.taptap.musicmanager.api.ApiException
import win.taptap.musicmanager.api.apiRoutes
import win.taptap.musicmanager.config.AppConfig
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.ClientLogRepository
import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.MoodRepository
@@ -184,6 +188,25 @@ fun main() {
}
}
// Client diagnostics are kept for a short window so a bug can be
// investigated without letting the table grow without bound.
applicationScope.launch(Dispatchers.IO) {
delay(TimeUnit.MINUTES.toMillis(1))
while (isActive) {
runCatching {
ClientLogRepository.prune(
config.clientLogRetentionDays,
config.clientLogMaxRowsPerUser,
)
}
.onSuccess { count ->
if (count > 0) log.info("Pruned {} client diagnostics rows", count)
}
.onFailure { log.warn("Client diagnostics prune failed", it) }
delay(TimeUnit.HOURS.toMillis(6))
}
}
// Delete files that no mood, recommendation or open session references.
applicationScope.launch(Dispatchers.IO) {
delay(TimeUnit.MINUTES.toMillis(10))
@@ -221,7 +244,15 @@ fun main() {
}
install(PartialContent)
install(AutoHeadResponse)
install(CallLogging)
// A short id on every response makes server log lines and client
// diagnostics meet in the middle.
install(CallId) {
generate { UUID.randomUUID().toString().take(8) }
replyToHeader("X-Request-Id")
}
install(CallLogging) {
callIdMdc("requestId")
}
install(StatusPages) {
exception<ApiException> { call, cause ->
call.respond(
@@ -271,6 +302,7 @@ fun main() {
deezerQuality = config.deezerQuality,
loudnessEnabled = config.loudnessEnabled && loudnessAnalyzer != null,
loudnessTargetLufs = config.loudnessTargetLufs,
clientLogRetentionDays = config.clientLogRetentionDays,
startScan = { scanner.start(applicationScope) },
scanStatus = { scanner.status() },
)
@@ -44,6 +44,49 @@ data class DeviceDto(
val current: Boolean,
)
@Serializable
data class ClientLogEntryDto(
/** Device clock in epoch milliseconds; informational, the server clamps it. */
val timestamp: Long = 0,
val level: String,
val tag: String,
val message: String,
val detail: String? = null,
val sessionId: String? = null,
)
@Serializable
data class ClientLogBatchDto(
val deviceId: String = "unknown",
val platform: String = "unknown",
val appVersion: String = "unknown",
val entries: List<ClientLogEntryDto> = emptyList(),
)
@Serializable
data class ClientLogDto(
val id: Long,
val deviceId: String,
val platform: String,
val appVersion: String,
val sessionId: String? = null,
val level: String,
val tag: String,
val message: String,
val detail: String? = null,
val createdAt: String,
val receivedAt: String,
)
@Serializable
data class ClientLogsDto(val logs: List<ClientLogDto>)
@Serializable
data class ClientLogsAcceptedDto(val accepted: Int)
@Serializable
data class ClientLogsDeletedDto(val deleted: Int)
@Serializable
data class ServerInfoDto(
val defaultSource: String,
@@ -51,6 +94,8 @@ data class ServerInfoDto(
val deezerQuality: Int,
val loudnessEnabled: Boolean,
val loudnessTargetLufs: Double,
/** How long the server keeps uploaded client diagnostics. */
val clientLogRetentionDays: Int,
)
@Serializable
@@ -40,6 +40,7 @@ fun Route.apiRoutes(
deezerQuality: Int,
loudnessEnabled: Boolean,
loudnessTargetLufs: Double,
clientLogRetentionDays: Int,
startScan: () -> Boolean,
scanStatus: () -> ScanStatus,
) {
@@ -89,6 +90,7 @@ fun Route.apiRoutes(
deezerQuality = deezerQuality,
loudnessEnabled = loudnessEnabled,
loudnessTargetLufs = loudnessTargetLufs,
clientLogRetentionDays = clientLogRetentionDays,
),
)
}
@@ -151,6 +153,7 @@ fun Route.apiRoutes(
defaultSource = defaultSource,
)
sessionRoutes(users, radioService, gc)
logRoutes(users)
}
}
@@ -0,0 +1,142 @@
package win.taptap.musicmanager.api
import io.ktor.http.HttpStatusCode
import io.ktor.server.application.ApplicationCall
import io.ktor.server.request.contentLength
import io.ktor.server.response.respond
import io.ktor.server.routing.Route
import io.ktor.server.routing.delete
import io.ktor.server.routing.get
import io.ktor.server.routing.post
import java.time.Instant
import java.time.temporal.ChronoUnit
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.config.UserAccount
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.ClientLogRepository
import win.taptap.musicmanager.db.ClientLogSanitizer
import win.taptap.musicmanager.db.toDto
private val log = LoggerFactory.getLogger("win.taptap.musicmanager.api.Logs")
/** A diagnostic batch is small; anything bigger is a bug or abuse. */
private const val MAX_BODY_BYTES = 2L * 1024 * 1024
/** Keep the insert transaction short and the per-request cost predictable. */
private const val MAX_ENTRIES_PER_BATCH = 200
/** Per-user ingest ceiling, measured over the last hour. */
private const val MAX_ROWS_PER_HOUR = 20_000
/**
* Client diagnostics endpoints.
*
* The app uploads warnings/errors and a few context entries in batches. Each
* row belongs to the authenticated user (or to the user an administrator names),
* so one account can never read another account's logs. The server redacts
* credentials again before storing anything.
*/
fun Route.logRoutes(users: UsersConfig) {
post("/logs") {
val user = call.requireApiUser(users)
val length = call.request.contentLength()
if (length != null && length > MAX_BODY_BYTES) {
throw ApiException(
HttpStatusCode.PayloadTooLarge,
"logs_too_large",
"A log batch may not exceed ${MAX_BODY_BYTES / 1024} KiB",
)
}
val batch = call.receiveJson<ClientLogBatchDto>()
if (batch.entries.isEmpty()) badRequest("A log batch needs at least one entry")
if (batch.entries.size > MAX_ENTRIES_PER_BATCH) {
badRequest("A log batch may contain at most $MAX_ENTRIES_PER_BATCH entries")
}
val payloadBytes = batch.entries.sumOf { entry ->
entry.message.length + (entry.detail?.length ?: 0) + entry.tag.length
}
if (payloadBytes > MAX_BODY_BYTES) {
throw ApiException(
HttpStatusCode.PayloadTooLarge,
"logs_too_large",
"A log batch may not exceed ${MAX_BODY_BYTES / 1024} KiB",
)
}
batch.entries.forEach { entry ->
if (ClientLogSanitizer.normalizeLevel(entry.level) == null) {
badRequest("Invalid log level: ${entry.level.take(16)}")
}
}
val since = Instant.now().minus(1, ChronoUnit.HOURS)
if (ClientLogRepository.countSince(user.name, since) >= MAX_ROWS_PER_HOUR) {
throw ApiException(
HttpStatusCode.TooManyRequests,
"logs_rate_limited",
"Too many diagnostic entries in the last hour",
)
}
val accepted = ClientLogRepository.insert(user.name, batch)
// Errors should show up in the server log immediately, not only when
// someone remembers to query the diagnostics endpoint.
batch.entries
.filter { ClientLogSanitizer.normalizeLevel(it.level) == "error" }
.take(3)
.forEach { entry ->
log.warn(
"Client error [{}] {}: {}",
user.name,
ClientLogSanitizer.tag(entry.tag),
ClientLogSanitizer.message(entry.message).take(300),
)
}
call.respond(ClientLogsAcceptedDto(accepted))
}
get("/logs") {
val user = call.requireApiUser(users)
val target = call.logTargetUser(users, user)
val level = call.request.queryParameters["level"]
?.takeIf { it.isNotBlank() }
?.let {
ClientLogSanitizer.normalizeLevel(it)
?: badRequest("level must be one of ${ClientLogSanitizer.levels.sorted().joinToString()}")
}
val deviceId = call.request.queryParameters["device"]?.takeIf { it.isNotBlank() }
val beforeId = call.request.queryParameters["before"]?.let { raw ->
raw.toLongOrNull()?.takeIf { it > 0 } ?: badRequest("before must be a positive row id")
}
val sinceMillis = call.request.queryParameters["since"]?.let { raw ->
raw.toLongOrNull()?.takeIf { it > 0 } ?: badRequest("since must be epoch milliseconds")
}
val limit = call.request.queryParameters["limit"]?.let { raw ->
raw.toIntOrNull()?.takeIf { it in 1..1_000 } ?: badRequest("limit must be between 1 and 1000")
} ?: 200
val logs = ClientLogRepository.list(
userName = target,
level = level,
deviceId = deviceId,
beforeId = beforeId,
sinceMillis = sinceMillis,
limit = limit,
)
call.respond(ClientLogsDto(logs.map { it.toDto() }))
}
delete("/logs") {
val user = call.requireApiUser(users)
val target = call.logTargetUser(users, user)
val deleted = ClientLogRepository.delete(target)
call.respond(ClientLogsDeletedDto(deleted))
}
}
private fun ApplicationCall.logTargetUser(users: UsersConfig, current: UserAccount): String {
val requested = request.queryParameters["user"]?.takeIf { it.isNotBlank() }
if (requested == null || requested == current.name) return current.name
if (!current.admin) forbidden("Only administrators can read another user's logs")
return users.byName(requested)?.name ?: notFound("User")
}
@@ -26,6 +26,10 @@ data class AppConfig(
val loudnessEnabled: Boolean = true,
/** Integrated loudness target; -16 LUFS is the ReplayGain-style value. */
val loudnessTargetLufs: Double = -16.0,
/** Client diagnostics: rows older than this are pruned. */
val clientLogRetentionDays: Int = 14,
/** Client diagnostics: newest rows kept per user. */
val clientLogMaxRowsPerUser: Int = 20_000,
val mediaDir: Path = dataDir.resolve("media"),
val downloadDir: Path = dataDir.resolve("downloads"),
) {
@@ -62,6 +66,18 @@ data class AppConfig(
require(loudnessTargetLufs in -30.0..-5.0) {
"LOUDNESS_TARGET_LUFS must be between -30 and -5, got: $loudnessTargetLufs"
}
val rawRetention = value("CLIENT_LOG_RETENTION_DAYS", "14")
val clientLogRetentionDays = rawRetention.toIntOrNull()
?: throw IllegalArgumentException("CLIENT_LOG_RETENTION_DAYS must be an integer")
require(clientLogRetentionDays in 1..365) {
"CLIENT_LOG_RETENTION_DAYS must be between 1 and 365, got: $clientLogRetentionDays"
}
val rawLogRows = value("CLIENT_LOG_MAX_ROWS_PER_USER", "20000")
val clientLogMaxRowsPerUser = rawLogRows.toIntOrNull()
?: throw IllegalArgumentException("CLIENT_LOG_MAX_ROWS_PER_USER must be an integer")
require(clientLogMaxRowsPerUser in 100..1_000_000) {
"CLIENT_LOG_MAX_ROWS_PER_USER must be between 100 and 1000000, got: $clientLogMaxRowsPerUser"
}
return AppConfig(
port = port,
@@ -76,6 +92,8 @@ data class AppConfig(
deezerApiBase = value("DEEZER_API_BASE", "https://api.deezer.com"),
loudnessEnabled = loudnessEnabled,
loudnessTargetLufs = loudnessTargetLufs,
clientLogRetentionDays = clientLogRetentionDays,
clientLogMaxRowsPerUser = clientLogMaxRowsPerUser,
mediaDir = Path.of(value("MEDIA_DIR", dataDir.resolve("media").toString())),
downloadDir = Path.of(value("DOWNLOAD_DIR", dataDir.resolve("downloads").toString())),
)
@@ -0,0 +1,234 @@
package win.taptap.musicmanager.db
import java.time.Instant
import java.time.temporal.ChronoUnit
import org.jetbrains.exposed.sql.Op
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.greaterEq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.less
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.deleteWhere
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import win.taptap.musicmanager.api.ClientLogBatchDto
import win.taptap.musicmanager.api.ClientLogDto
data class ClientLogRecord(
val id: Long,
val deviceId: String,
val platform: String,
val appVersion: String,
val sessionId: String?,
val level: String,
val tag: String,
val message: String,
val detail: String?,
val createdAt: String,
val receivedAt: String,
)
fun ClientLogRecord.toDto(): ClientLogDto = ClientLogDto(
id = id,
deviceId = deviceId,
platform = platform,
appVersion = appVersion,
sessionId = sessionId,
level = level,
tag = tag,
message = message,
detail = detail,
createdAt = createdAt,
receivedAt = receivedAt,
)
/**
* Removes credentials before anything is persisted. The client redacts as well,
* but the server must not trust a buggy or older build: a token in a log line
* would give whoever reads the table access to the account.
*/
object ClientLogSanitizer {
private const val MAX_MESSAGE = 4_000
private const val MAX_DETAIL = 16_000
private const val MAX_TAG = 64
private const val MAX_DEVICE_ID = 80
private const val MAX_PLATFORM = 32
private const val MAX_APP_VERSION = 40
private const val MAX_SESSION_ID = 64
private val bearer = Regex("""(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]{8,}""")
private val authorization = Regex("""(?i)\b(authorization\s*:\s*)[^\r\n]+""")
private val querySecret = Regex(
"""(?i)(\b(?:token|access_token|password|passwd|arl|key)=)[^&\s"'<>]+""",
)
private val jsonSecret = Regex(
"""(?i)("(?:token|access_token|password|arl|key)"\s*:\s*")[^"]*(")""",
)
val levels = setOf("debug", "info", "warn", "error")
fun redact(value: String): String = value
.replace(bearer) { match -> match.groupValues[1] + "***" }
.replace(authorization) { match -> match.groupValues[1] + "***" }
.replace(querySecret) { match -> match.groupValues[1] + "***" }
.replace(jsonSecret) { match -> match.groupValues[1] + "***" + match.groupValues[2] }
fun normalizeLevel(value: String): String? =
value.trim().lowercase().takeIf { it in levels }
fun deviceId(value: String): String = clean(value, MAX_DEVICE_ID).ifBlank { "unknown" }
fun platform(value: String): String = clean(value, MAX_PLATFORM).ifBlank { "unknown" }
fun appVersion(value: String): String = clean(value, MAX_APP_VERSION).ifBlank { "unknown" }
fun sessionId(value: String?): String? =
value?.let { clean(it, MAX_SESSION_ID) }?.takeIf { it.isNotBlank() }
fun tag(value: String): String = clean(value, MAX_TAG).ifBlank { "app" }
fun message(value: String): String = clean(value, MAX_MESSAGE).ifBlank { "(empty)" }
fun detail(value: String?): String? =
value?.let { clean(it, MAX_DETAIL) }?.takeIf { it.isNotBlank() }
private fun clean(value: String, max: Int): String =
redact(value).trim().let { if (it.length > max) it.take(max) else it }
}
/**
* Client diagnostics storage. A batch is written in one transaction so a broken
* request cannot leave half a log behind, and retention runs on the server clock.
*/
object ClientLogRepository {
private const val MAX_CLOCK_SKEW_MILLIS = 5 * 60_000L
private const val MAX_ENTRY_AGE_MILLIS = 90L * 24 * 60 * 60_000
fun insert(userName: String, batch: ClientLogBatchDto): Int {
val now = Instant.now()
val receivedAt = now.toString()
val deviceId = ClientLogSanitizer.deviceId(batch.deviceId)
val platform = ClientLogSanitizer.platform(batch.platform)
val appVersion = ClientLogSanitizer.appVersion(batch.appVersion)
return transaction {
var inserted = 0
batch.entries.forEach { entry ->
val level = ClientLogSanitizer.normalizeLevel(entry.level) ?: return@forEach
val millis = entry.timestamp
.takeIf { it > 0 }
?.coerceIn(
now.toEpochMilli() - MAX_ENTRY_AGE_MILLIS,
now.toEpochMilli() + MAX_CLOCK_SKEW_MILLIS,
)
?: now.toEpochMilli()
ClientLogsTable.insert {
it[ClientLogsTable.userName] = userName
it[ClientLogsTable.deviceId] = deviceId
it[ClientLogsTable.platform] = platform
it[ClientLogsTable.appVersion] = appVersion
it[sessionId] = ClientLogSanitizer.sessionId(entry.sessionId)
it[ClientLogsTable.level] = level
it[tag] = ClientLogSanitizer.tag(entry.tag)
it[message] = ClientLogSanitizer.message(entry.message)
it[detail] = ClientLogSanitizer.detail(entry.detail)
it[createdAt] = Instant.ofEpochMilli(millis).toString()
it[ClientLogsTable.receivedAt] = receivedAt
}
inserted += 1
}
inserted
}
}
fun list(
userName: String,
level: String? = null,
deviceId: String? = null,
beforeId: Long? = null,
sinceMillis: Long? = null,
limit: Int = 200,
): List<ClientLogRecord> {
var condition: Op<Boolean> = ClientLogsTable.userName eq userName
level?.let { condition = condition and (ClientLogsTable.level eq it) }
deviceId?.let { condition = condition and (ClientLogsTable.deviceId eq it) }
beforeId?.let { condition = condition and (ClientLogsTable.id less it) }
sinceMillis?.let {
val since = Instant.ofEpochMilli(it).toString()
condition = condition and (ClientLogsTable.receivedAt greaterEq since)
}
return transaction {
ClientLogsTable.selectAll()
.where(condition)
.orderBy(ClientLogsTable.id to SortOrder.DESC)
.limit(limit.coerceIn(1, 1_000))
.map { it.toRecord() }
}
}
fun countSince(userName: String, since: Instant): Long = transaction {
ClientLogsTable.selectAll()
.where {
(ClientLogsTable.userName eq userName) and
(ClientLogsTable.receivedAt greaterEq since.toString())
}
.count()
}
fun delete(userName: String): Int = transaction {
ClientLogsTable.deleteWhere { ClientLogsTable.userName eq userName }
}
/**
* Deletes rows older than [retentionDays] and trims every user to
* [maxRowsPerUser], newest first. Returns the number of deleted rows.
*/
fun prune(retentionDays: Int, maxRowsPerUser: Int): Int {
val cutoff = Instant.now()
.minus(retentionDays.coerceAtLeast(1).toLong(), ChronoUnit.DAYS)
.toString()
var deleted = transaction {
ClientLogsTable.deleteWhere { ClientLogsTable.receivedAt less cutoff }
}
val users = transaction {
ClientLogsTable.select(ClientLogsTable.userName)
.withDistinct()
.map { it[ClientLogsTable.userName] }
}
users.forEach { user -> deleted += trim(user, maxRowsPerUser) }
return deleted
}
private fun trim(userName: String, maxRows: Int): Int {
if (maxRows <= 0) return delete(userName)
return transaction {
val cutoff = ClientLogsTable.select(ClientLogsTable.id)
.where { ClientLogsTable.userName eq userName }
.orderBy(ClientLogsTable.id to SortOrder.DESC)
.limit(1).offset((maxRows - 1).toLong())
.map { it[ClientLogsTable.id] }
.firstOrNull() ?: return@transaction 0
ClientLogsTable.deleteWhere {
(ClientLogsTable.userName eq userName) and
(ClientLogsTable.id less cutoff)
}
}
}
private fun ResultRow.toRecord(): ClientLogRecord = ClientLogRecord(
id = this[ClientLogsTable.id],
deviceId = this[ClientLogsTable.deviceId],
platform = this[ClientLogsTable.platform],
appVersion = this[ClientLogsTable.appVersion],
sessionId = this[ClientLogsTable.sessionId],
level = this[ClientLogsTable.level],
tag = this[ClientLogsTable.tag],
message = this[ClientLogsTable.message],
detail = this[ClientLogsTable.detail],
createdAt = this[ClientLogsTable.createdAt],
receivedAt = this[ClientLogsTable.receivedAt],
)
}
@@ -163,3 +163,24 @@ object SongAffinityTable : Table("song_affinity") {
override val primaryKey = PrimaryKey(userName, songId)
}
/**
* Batches of client diagnostics. `received_at` is the server clock and is what
* retention uses; `created_at` is the device clock and is only for display.
*/
object ClientLogsTable : Table("client_logs") {
val id = long("id").autoIncrement()
val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE)
val deviceId = varchar("device_id", 80)
val platform = varchar("platform", 32)
val appVersion = varchar("app_version", 40)
val sessionId = varchar("session_id", 64).nullable()
val level = varchar("level", 8)
val tag = varchar("tag", 64)
val message = text("message")
val detail = text("detail").nullable()
val createdAt = varchar("created_at", 40)
val receivedAt = varchar("received_at", 40)
override val primaryKey = PrimaryKey(id)
}