feat: accept, store and expose client diagnostics
CI / chart (push) Successful in 3s
CI / test (push) Successful in 2m20s
CI / chart-push (push) Successful in 6s
CI / image (push) Successful in 3m39s

- V11 client_logs table with per-user storage and retention/pruning
- POST/GET/DELETE /logs with batch validation, rate limit and admin access
- redact bearer tokens, Authorization headers and query/JSON secrets twice
- add X-Request-Id to responses and include it in the server log pattern
- report client error entries immediately in the server log
- scripts/logs.sh for source-side retrieval and clearing
- chart knobs CLIENT_LOG_RETENTION_DAYS and CLIENT_LOG_MAX_ROWS_PER_USER
This commit is contained in:
TapTap committed 2026-11-20 13:43:05 +00:00
1 parent 91312faa1c
commit 44b75c4d4d
19 files changed
+981 -5

No files matched your search

@@ -94,6 +94,7 @@ class ApiRoutesTest {
assertEquals(0, info["deezerQuality"]!!.jsonPrimitive.int)
assertTrue(info["loudnessEnabled"]!!.jsonPrimitive.content.toBoolean())
assertEquals("-16.0", info["loudnessTargetLufs"]!!.jsonPrimitive.content)
assertEquals(14, info["clientLogRetentionDays"]!!.jsonPrimitive.int)
val malformed = client.post("/api/v1/moods") {
bearer(token)
@@ -105,6 +106,88 @@ class ApiRoutesTest {
}
}
@Test
fun `client diagnostics are private and redacted`() {
seededContext().use { seed ->
testApplication {
setupApp(seed)
val alice = client.token("alice", "alice-pw")
val bob = client.token("bob", "bob-pw")
val batch = """
{"deviceId":"pixel-7","platform":"android","appVersion":"0.4.3",
"entries":[
{"timestamp":1700000000000,"level":"warn","tag":"playback","message":"stalled","detail":"Authorization: Bearer keep-secret","sessionId":"run-1"},
{"timestamp":1700000000000,"level":"error","tag":"api","message":"GET /stream -> 500","detail":"https://music/stream?token=abc123"}
]}
""".trimIndent()
val upload = client.post("/api/v1/logs") {
bearer(alice)
contentType(ContentType.Application.Json)
setBody(batch)
}
assertEquals(HttpStatusCode.OK, upload.status)
assertEquals(2, parse(upload.bodyAsText())["accepted"]!!.jsonPrimitive.int)
val list = parse(client.get("/api/v1/logs") { bearer(alice) }.bodyAsText())
val logs = list["logs"]!!.jsonArray
assertEquals(2, logs.size)
// Newest first: the error entry was inserted last.
assertEquals("api", logs[0].jsonObject.string("tag"))
assertEquals("error", logs[0].jsonObject.string("level"))
assertFalse(logs[0].jsonObject.string("detail").contains("abc123"))
assertFalse(logs[1].jsonObject.string("detail").contains("keep-secret"))
assertTrue(logs[1].jsonObject.string("detail").contains("***"))
val levelFiltered = parse(
client.get("/api/v1/logs?level=warn") { bearer(alice) }.bodyAsText(),
)
assertEquals(1, levelFiltered["logs"]!!.jsonArray.size)
val bobLogs = parse(client.get("/api/v1/logs") { bearer(bob) }.bodyAsText())
assertTrue(bobLogs["logs"]!!.jsonArray.isEmpty())
val forbidden = client.get("/api/v1/logs?user=alice") { bearer(bob) }
assertEquals(HttpStatusCode.Forbidden, forbidden.status)
val adminView = parse(
client.get("/api/v1/logs?user=alice") { bearer(alice) }.bodyAsText(),
)
assertEquals(2, adminView["logs"]!!.jsonArray.size)
val invalid = client.post("/api/v1/logs") {
bearer(alice)
contentType(ContentType.Application.Json)
setBody(
"""{"deviceId":"d","platform":"android","appVersion":"1",
"entries":[{"level":"verbose","tag":"x","message":"y"}]}""",
)
}
assertEquals(HttpStatusCode.BadRequest, invalid.status)
val many = (1..201).joinToString(",") {
"""{"level":"info","tag":"x","message":"m"}"""
}
val tooMany = client.post("/api/v1/logs") {
bearer(alice)
contentType(ContentType.Application.Json)
setBody(
"""{"deviceId":"d","platform":"android","appVersion":"1","entries":[$many]}""",
)
}
assertEquals(HttpStatusCode.BadRequest, tooMany.status)
val cleared = client.delete("/api/v1/logs") { bearer(alice) }
assertEquals(HttpStatusCode.OK, cleared.status)
assertEquals(2, parse(cleared.bodyAsText())["deleted"]!!.jsonPrimitive.int)
assertTrue(
parse(client.get("/api/v1/logs") { bearer(alice) }.bodyAsText())["logs"]!!
.jsonArray.isEmpty(),
)
}
}
}
@Test
fun `moods own their songs and are private`() {
seededContext().use { seed ->
@@ -644,6 +727,7 @@ class ApiRoutesTest {
deezerQuality = 0,
loudnessEnabled = true,
loudnessTargetLufs = -16.0,
clientLogRetentionDays = 14,
startScan = { false },
scanStatus = { ScanStatus(false, 0) },
)
@@ -0,0 +1,110 @@
package win.taptap.musicmanager.db
import java.nio.file.Files
import java.time.Instant
import java.time.temporal.ChronoUnit
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import win.taptap.musicmanager.api.ClientLogBatchDto
import win.taptap.musicmanager.api.ClientLogEntryDto
import win.taptap.musicmanager.config.AppConfig
import win.taptap.musicmanager.config.UserAccount
import win.taptap.musicmanager.config.UsersConfig
class ClientLogRepositoryTest {
@Test
fun `credentials are redacted before they are stored`() {
withDatabase { db ->
assertEquals(1, ClientLogRepository.insert("alice", batch("playback", "Authorization: Bearer supersecret")))
val row = ClientLogRepository.list("alice").single()
assertFalse(row.message.contains("supersecret"))
assertFalse(row.detail.orEmpty().contains("supersecret"))
assertTrue(row.detail.orEmpty().contains("***"))
}
}
@Test
fun `retention removes old rows and the per-user cap keeps the newest`() {
withDatabase { db ->
ClientLogRepository.insert("alice", batch("one", "first"))
ClientLogRepository.insert("alice", batch("two", "second"))
ClientLogRepository.insert("alice", batch("three", "third"))
// Three rows, keep two: the newest two survive.
assertEquals(1, ClientLogRepository.prune(365, 2))
assertEquals(listOf("third", "second"), ClientLogRepository.list("alice").map { it.message })
// Age the remaining rows past a one-day retention window.
transaction {
ClientLogsTable.update({ ClientLogsTable.userName eq "alice" }) {
it[receivedAt] = Instant.now().minus(10, ChronoUnit.DAYS).toString()
}
}
assertEquals(2, ClientLogRepository.prune(7, 2))
assertTrue(ClientLogRepository.list("alice").isEmpty())
}
}
@Test
fun `users cannot see each other's diagnostics`() {
withDatabase { db ->
ClientLogRepository.insert("alice", batch("a", "alice line"))
ClientLogRepository.insert("bob", batch("b", "bob line"))
assertEquals(listOf("alice line"), ClientLogRepository.list("alice").map { it.message })
assertEquals(listOf("bob line"), ClientLogRepository.list("bob").map { it.message })
assertEquals(1, ClientLogRepository.delete("alice"))
assertTrue(ClientLogRepository.list("alice").isEmpty())
assertEquals(1, ClientLogRepository.list("bob").size)
}
}
private fun withDatabase(block: (DatabaseHandle) -> Unit) {
val dir = Files.createTempDirectory("client-logs-test")
val users = UsersConfig(
listOf(
UserAccount("alice", "alice-pw", admin = true),
UserAccount("bob", "bob-pw", admin = false),
),
)
val db = DatabaseFactory.connect(
AppConfig(
port = 0,
dataDir = dir,
usersFile = dir.resolve("users.yaml"),
ffmpegPath = "ffmpeg",
),
)
try {
ConfigRepository.sync(users)
block(db)
} finally {
db.close()
}
}
private fun batch(tag: String, message: String): ClientLogBatchDto =
ClientLogBatchDto(
deviceId = "test-device",
platform = "desktop",
appVersion = "0.4.3",
entries = listOf(
ClientLogEntryDto(
timestamp = System.currentTimeMillis(),
level = "warn",
tag = tag,
message = message,
detail = "request failed: Authorization: Bearer supersecret",
sessionId = "run-test",
),
),
)
}