feat: accept, store and expose client diagnostics
- V11 client_logs table with per-user storage and retention/pruning - POST/GET/DELETE /logs with batch validation, rate limit and admin access - redact bearer tokens, Authorization headers and query/JSON secrets twice - add X-Request-Id to responses and include it in the server log pattern - report client error entries immediately in the server log - scripts/logs.sh for source-side retrieval and clearing - chart knobs CLIENT_LOG_RETENTION_DAYS and CLIENT_LOG_MAX_ROWS_PER_USER
This commit is contained in:
1 parent
91312faa1c
commit
44b75c4d4d
19 files changed
+981
-5
No files matched your search
@@ -9,6 +9,8 @@ import io.ktor.server.application.install
|
||||
import io.ktor.server.engine.embeddedServer
|
||||
import io.ktor.server.netty.Netty
|
||||
import io.ktor.server.plugins.autohead.AutoHeadResponse
|
||||
import io.ktor.server.plugins.callid.CallId
|
||||
import io.ktor.server.plugins.callid.callIdMdc
|
||||
import io.ktor.server.plugins.calllogging.CallLogging
|
||||
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
|
||||
import io.ktor.server.plugins.cors.routing.CORS
|
||||
@@ -20,6 +22,7 @@ import io.ktor.server.response.respondText
|
||||
import io.ktor.server.routing.get
|
||||
import io.ktor.server.routing.routing
|
||||
import java.nio.file.Files
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
@@ -38,6 +41,7 @@ import win.taptap.musicmanager.api.ApiException
|
||||
import win.taptap.musicmanager.api.apiRoutes
|
||||
import win.taptap.musicmanager.config.AppConfig
|
||||
import win.taptap.musicmanager.config.UsersConfig
|
||||
import win.taptap.musicmanager.db.ClientLogRepository
|
||||
import win.taptap.musicmanager.db.ConfigRepository
|
||||
import win.taptap.musicmanager.db.DatabaseFactory
|
||||
import win.taptap.musicmanager.db.MoodRepository
|
||||
@@ -184,6 +188,25 @@ fun main() {
|
||||
}
|
||||
}
|
||||
|
||||
// Client diagnostics are kept for a short window so a bug can be
|
||||
// investigated without letting the table grow without bound.
|
||||
applicationScope.launch(Dispatchers.IO) {
|
||||
delay(TimeUnit.MINUTES.toMillis(1))
|
||||
while (isActive) {
|
||||
runCatching {
|
||||
ClientLogRepository.prune(
|
||||
config.clientLogRetentionDays,
|
||||
config.clientLogMaxRowsPerUser,
|
||||
)
|
||||
}
|
||||
.onSuccess { count ->
|
||||
if (count > 0) log.info("Pruned {} client diagnostics rows", count)
|
||||
}
|
||||
.onFailure { log.warn("Client diagnostics prune failed", it) }
|
||||
delay(TimeUnit.HOURS.toMillis(6))
|
||||
}
|
||||
}
|
||||
|
||||
// Delete files that no mood, recommendation or open session references.
|
||||
applicationScope.launch(Dispatchers.IO) {
|
||||
delay(TimeUnit.MINUTES.toMillis(10))
|
||||
@@ -221,7 +244,15 @@ fun main() {
|
||||
}
|
||||
install(PartialContent)
|
||||
install(AutoHeadResponse)
|
||||
install(CallLogging)
|
||||
// A short id on every response makes server log lines and client
|
||||
// diagnostics meet in the middle.
|
||||
install(CallId) {
|
||||
generate { UUID.randomUUID().toString().take(8) }
|
||||
replyToHeader("X-Request-Id")
|
||||
}
|
||||
install(CallLogging) {
|
||||
callIdMdc("requestId")
|
||||
}
|
||||
install(StatusPages) {
|
||||
exception<ApiException> { call, cause ->
|
||||
call.respond(
|
||||
@@ -271,6 +302,7 @@ fun main() {
|
||||
deezerQuality = config.deezerQuality,
|
||||
loudnessEnabled = config.loudnessEnabled && loudnessAnalyzer != null,
|
||||
loudnessTargetLufs = config.loudnessTargetLufs,
|
||||
clientLogRetentionDays = config.clientLogRetentionDays,
|
||||
startScan = { scanner.start(applicationScope) },
|
||||
scanStatus = { scanner.status() },
|
||||
)
|
||||
|
||||
@@ -44,6 +44,49 @@ data class DeviceDto(
|
||||
val current: Boolean,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class ClientLogEntryDto(
|
||||
/** Device clock in epoch milliseconds; informational, the server clamps it. */
|
||||
val timestamp: Long = 0,
|
||||
val level: String,
|
||||
val tag: String,
|
||||
val message: String,
|
||||
val detail: String? = null,
|
||||
val sessionId: String? = null,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class ClientLogBatchDto(
|
||||
val deviceId: String = "unknown",
|
||||
val platform: String = "unknown",
|
||||
val appVersion: String = "unknown",
|
||||
val entries: List<ClientLogEntryDto> = emptyList(),
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class ClientLogDto(
|
||||
val id: Long,
|
||||
val deviceId: String,
|
||||
val platform: String,
|
||||
val appVersion: String,
|
||||
val sessionId: String? = null,
|
||||
val level: String,
|
||||
val tag: String,
|
||||
val message: String,
|
||||
val detail: String? = null,
|
||||
val createdAt: String,
|
||||
val receivedAt: String,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class ClientLogsDto(val logs: List<ClientLogDto>)
|
||||
|
||||
@Serializable
|
||||
data class ClientLogsAcceptedDto(val accepted: Int)
|
||||
|
||||
@Serializable
|
||||
data class ClientLogsDeletedDto(val deleted: Int)
|
||||
|
||||
@Serializable
|
||||
data class ServerInfoDto(
|
||||
val defaultSource: String,
|
||||
@@ -51,6 +94,8 @@ data class ServerInfoDto(
|
||||
val deezerQuality: Int,
|
||||
val loudnessEnabled: Boolean,
|
||||
val loudnessTargetLufs: Double,
|
||||
/** How long the server keeps uploaded client diagnostics. */
|
||||
val clientLogRetentionDays: Int,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
|
||||
@@ -40,6 +40,7 @@ fun Route.apiRoutes(
|
||||
deezerQuality: Int,
|
||||
loudnessEnabled: Boolean,
|
||||
loudnessTargetLufs: Double,
|
||||
clientLogRetentionDays: Int,
|
||||
startScan: () -> Boolean,
|
||||
scanStatus: () -> ScanStatus,
|
||||
) {
|
||||
@@ -89,6 +90,7 @@ fun Route.apiRoutes(
|
||||
deezerQuality = deezerQuality,
|
||||
loudnessEnabled = loudnessEnabled,
|
||||
loudnessTargetLufs = loudnessTargetLufs,
|
||||
clientLogRetentionDays = clientLogRetentionDays,
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -151,6 +153,7 @@ fun Route.apiRoutes(
|
||||
defaultSource = defaultSource,
|
||||
)
|
||||
sessionRoutes(users, radioService, gc)
|
||||
logRoutes(users)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package win.taptap.musicmanager.api
|
||||
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import io.ktor.server.application.ApplicationCall
|
||||
import io.ktor.server.request.contentLength
|
||||
import io.ktor.server.response.respond
|
||||
import io.ktor.server.routing.Route
|
||||
import io.ktor.server.routing.delete
|
||||
import io.ktor.server.routing.get
|
||||
import io.ktor.server.routing.post
|
||||
import java.time.Instant
|
||||
import java.time.temporal.ChronoUnit
|
||||
import org.slf4j.LoggerFactory
|
||||
import win.taptap.musicmanager.config.UserAccount
|
||||
import win.taptap.musicmanager.config.UsersConfig
|
||||
import win.taptap.musicmanager.db.ClientLogRepository
|
||||
import win.taptap.musicmanager.db.ClientLogSanitizer
|
||||
import win.taptap.musicmanager.db.toDto
|
||||
|
||||
private val log = LoggerFactory.getLogger("win.taptap.musicmanager.api.Logs")
|
||||
|
||||
/** A diagnostic batch is small; anything bigger is a bug or abuse. */
|
||||
private const val MAX_BODY_BYTES = 2L * 1024 * 1024
|
||||
|
||||
/** Keep the insert transaction short and the per-request cost predictable. */
|
||||
private const val MAX_ENTRIES_PER_BATCH = 200
|
||||
|
||||
/** Per-user ingest ceiling, measured over the last hour. */
|
||||
private const val MAX_ROWS_PER_HOUR = 20_000
|
||||
|
||||
/**
|
||||
* Client diagnostics endpoints.
|
||||
*
|
||||
* The app uploads warnings/errors and a few context entries in batches. Each
|
||||
* row belongs to the authenticated user (or to the user an administrator names),
|
||||
* so one account can never read another account's logs. The server redacts
|
||||
* credentials again before storing anything.
|
||||
*/
|
||||
fun Route.logRoutes(users: UsersConfig) {
|
||||
|
||||
post("/logs") {
|
||||
val user = call.requireApiUser(users)
|
||||
val length = call.request.contentLength()
|
||||
if (length != null && length > MAX_BODY_BYTES) {
|
||||
throw ApiException(
|
||||
HttpStatusCode.PayloadTooLarge,
|
||||
"logs_too_large",
|
||||
"A log batch may not exceed ${MAX_BODY_BYTES / 1024} KiB",
|
||||
)
|
||||
}
|
||||
val batch = call.receiveJson<ClientLogBatchDto>()
|
||||
if (batch.entries.isEmpty()) badRequest("A log batch needs at least one entry")
|
||||
if (batch.entries.size > MAX_ENTRIES_PER_BATCH) {
|
||||
badRequest("A log batch may contain at most $MAX_ENTRIES_PER_BATCH entries")
|
||||
}
|
||||
val payloadBytes = batch.entries.sumOf { entry ->
|
||||
entry.message.length + (entry.detail?.length ?: 0) + entry.tag.length
|
||||
}
|
||||
if (payloadBytes > MAX_BODY_BYTES) {
|
||||
throw ApiException(
|
||||
HttpStatusCode.PayloadTooLarge,
|
||||
"logs_too_large",
|
||||
"A log batch may not exceed ${MAX_BODY_BYTES / 1024} KiB",
|
||||
)
|
||||
}
|
||||
batch.entries.forEach { entry ->
|
||||
if (ClientLogSanitizer.normalizeLevel(entry.level) == null) {
|
||||
badRequest("Invalid log level: ${entry.level.take(16)}")
|
||||
}
|
||||
}
|
||||
|
||||
val since = Instant.now().minus(1, ChronoUnit.HOURS)
|
||||
if (ClientLogRepository.countSince(user.name, since) >= MAX_ROWS_PER_HOUR) {
|
||||
throw ApiException(
|
||||
HttpStatusCode.TooManyRequests,
|
||||
"logs_rate_limited",
|
||||
"Too many diagnostic entries in the last hour",
|
||||
)
|
||||
}
|
||||
|
||||
val accepted = ClientLogRepository.insert(user.name, batch)
|
||||
// Errors should show up in the server log immediately, not only when
|
||||
// someone remembers to query the diagnostics endpoint.
|
||||
batch.entries
|
||||
.filter { ClientLogSanitizer.normalizeLevel(it.level) == "error" }
|
||||
.take(3)
|
||||
.forEach { entry ->
|
||||
log.warn(
|
||||
"Client error [{}] {}: {}",
|
||||
user.name,
|
||||
ClientLogSanitizer.tag(entry.tag),
|
||||
ClientLogSanitizer.message(entry.message).take(300),
|
||||
)
|
||||
}
|
||||
call.respond(ClientLogsAcceptedDto(accepted))
|
||||
}
|
||||
|
||||
get("/logs") {
|
||||
val user = call.requireApiUser(users)
|
||||
val target = call.logTargetUser(users, user)
|
||||
val level = call.request.queryParameters["level"]
|
||||
?.takeIf { it.isNotBlank() }
|
||||
?.let {
|
||||
ClientLogSanitizer.normalizeLevel(it)
|
||||
?: badRequest("level must be one of ${ClientLogSanitizer.levels.sorted().joinToString()}")
|
||||
}
|
||||
val deviceId = call.request.queryParameters["device"]?.takeIf { it.isNotBlank() }
|
||||
val beforeId = call.request.queryParameters["before"]?.let { raw ->
|
||||
raw.toLongOrNull()?.takeIf { it > 0 } ?: badRequest("before must be a positive row id")
|
||||
}
|
||||
val sinceMillis = call.request.queryParameters["since"]?.let { raw ->
|
||||
raw.toLongOrNull()?.takeIf { it > 0 } ?: badRequest("since must be epoch milliseconds")
|
||||
}
|
||||
val limit = call.request.queryParameters["limit"]?.let { raw ->
|
||||
raw.toIntOrNull()?.takeIf { it in 1..1_000 } ?: badRequest("limit must be between 1 and 1000")
|
||||
} ?: 200
|
||||
|
||||
val logs = ClientLogRepository.list(
|
||||
userName = target,
|
||||
level = level,
|
||||
deviceId = deviceId,
|
||||
beforeId = beforeId,
|
||||
sinceMillis = sinceMillis,
|
||||
limit = limit,
|
||||
)
|
||||
call.respond(ClientLogsDto(logs.map { it.toDto() }))
|
||||
}
|
||||
|
||||
delete("/logs") {
|
||||
val user = call.requireApiUser(users)
|
||||
val target = call.logTargetUser(users, user)
|
||||
val deleted = ClientLogRepository.delete(target)
|
||||
call.respond(ClientLogsDeletedDto(deleted))
|
||||
}
|
||||
}
|
||||
|
||||
private fun ApplicationCall.logTargetUser(users: UsersConfig, current: UserAccount): String {
|
||||
val requested = request.queryParameters["user"]?.takeIf { it.isNotBlank() }
|
||||
if (requested == null || requested == current.name) return current.name
|
||||
if (!current.admin) forbidden("Only administrators can read another user's logs")
|
||||
return users.byName(requested)?.name ?: notFound("User")
|
||||
}
|
||||
@@ -26,6 +26,10 @@ data class AppConfig(
|
||||
val loudnessEnabled: Boolean = true,
|
||||
/** Integrated loudness target; -16 LUFS is the ReplayGain-style value. */
|
||||
val loudnessTargetLufs: Double = -16.0,
|
||||
/** Client diagnostics: rows older than this are pruned. */
|
||||
val clientLogRetentionDays: Int = 14,
|
||||
/** Client diagnostics: newest rows kept per user. */
|
||||
val clientLogMaxRowsPerUser: Int = 20_000,
|
||||
val mediaDir: Path = dataDir.resolve("media"),
|
||||
val downloadDir: Path = dataDir.resolve("downloads"),
|
||||
) {
|
||||
@@ -62,6 +66,18 @@ data class AppConfig(
|
||||
require(loudnessTargetLufs in -30.0..-5.0) {
|
||||
"LOUDNESS_TARGET_LUFS must be between -30 and -5, got: $loudnessTargetLufs"
|
||||
}
|
||||
val rawRetention = value("CLIENT_LOG_RETENTION_DAYS", "14")
|
||||
val clientLogRetentionDays = rawRetention.toIntOrNull()
|
||||
?: throw IllegalArgumentException("CLIENT_LOG_RETENTION_DAYS must be an integer")
|
||||
require(clientLogRetentionDays in 1..365) {
|
||||
"CLIENT_LOG_RETENTION_DAYS must be between 1 and 365, got: $clientLogRetentionDays"
|
||||
}
|
||||
val rawLogRows = value("CLIENT_LOG_MAX_ROWS_PER_USER", "20000")
|
||||
val clientLogMaxRowsPerUser = rawLogRows.toIntOrNull()
|
||||
?: throw IllegalArgumentException("CLIENT_LOG_MAX_ROWS_PER_USER must be an integer")
|
||||
require(clientLogMaxRowsPerUser in 100..1_000_000) {
|
||||
"CLIENT_LOG_MAX_ROWS_PER_USER must be between 100 and 1000000, got: $clientLogMaxRowsPerUser"
|
||||
}
|
||||
|
||||
return AppConfig(
|
||||
port = port,
|
||||
@@ -76,6 +92,8 @@ data class AppConfig(
|
||||
deezerApiBase = value("DEEZER_API_BASE", "https://api.deezer.com"),
|
||||
loudnessEnabled = loudnessEnabled,
|
||||
loudnessTargetLufs = loudnessTargetLufs,
|
||||
clientLogRetentionDays = clientLogRetentionDays,
|
||||
clientLogMaxRowsPerUser = clientLogMaxRowsPerUser,
|
||||
mediaDir = Path.of(value("MEDIA_DIR", dataDir.resolve("media").toString())),
|
||||
downloadDir = Path.of(value("DOWNLOAD_DIR", dataDir.resolve("downloads").toString())),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
package win.taptap.musicmanager.db
|
||||
|
||||
import java.time.Instant
|
||||
import java.time.temporal.ChronoUnit
|
||||
import org.jetbrains.exposed.sql.Op
|
||||
import org.jetbrains.exposed.sql.ResultRow
|
||||
import org.jetbrains.exposed.sql.SortOrder
|
||||
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
|
||||
import org.jetbrains.exposed.sql.SqlExpressionBuilder.greaterEq
|
||||
import org.jetbrains.exposed.sql.SqlExpressionBuilder.less
|
||||
import org.jetbrains.exposed.sql.and
|
||||
import org.jetbrains.exposed.sql.deleteWhere
|
||||
import org.jetbrains.exposed.sql.insert
|
||||
import org.jetbrains.exposed.sql.selectAll
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import win.taptap.musicmanager.api.ClientLogBatchDto
|
||||
import win.taptap.musicmanager.api.ClientLogDto
|
||||
|
||||
data class ClientLogRecord(
|
||||
val id: Long,
|
||||
val deviceId: String,
|
||||
val platform: String,
|
||||
val appVersion: String,
|
||||
val sessionId: String?,
|
||||
val level: String,
|
||||
val tag: String,
|
||||
val message: String,
|
||||
val detail: String?,
|
||||
val createdAt: String,
|
||||
val receivedAt: String,
|
||||
)
|
||||
|
||||
fun ClientLogRecord.toDto(): ClientLogDto = ClientLogDto(
|
||||
id = id,
|
||||
deviceId = deviceId,
|
||||
platform = platform,
|
||||
appVersion = appVersion,
|
||||
sessionId = sessionId,
|
||||
level = level,
|
||||
tag = tag,
|
||||
message = message,
|
||||
detail = detail,
|
||||
createdAt = createdAt,
|
||||
receivedAt = receivedAt,
|
||||
)
|
||||
|
||||
/**
|
||||
* Removes credentials before anything is persisted. The client redacts as well,
|
||||
* but the server must not trust a buggy or older build: a token in a log line
|
||||
* would give whoever reads the table access to the account.
|
||||
*/
|
||||
object ClientLogSanitizer {
|
||||
|
||||
private const val MAX_MESSAGE = 4_000
|
||||
private const val MAX_DETAIL = 16_000
|
||||
private const val MAX_TAG = 64
|
||||
private const val MAX_DEVICE_ID = 80
|
||||
private const val MAX_PLATFORM = 32
|
||||
private const val MAX_APP_VERSION = 40
|
||||
private const val MAX_SESSION_ID = 64
|
||||
|
||||
private val bearer = Regex("""(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]{8,}""")
|
||||
private val authorization = Regex("""(?i)\b(authorization\s*:\s*)[^\r\n]+""")
|
||||
private val querySecret = Regex(
|
||||
"""(?i)(\b(?:token|access_token|password|passwd|arl|key)=)[^&\s"'<>]+""",
|
||||
)
|
||||
private val jsonSecret = Regex(
|
||||
"""(?i)("(?:token|access_token|password|arl|key)"\s*:\s*")[^"]*(")""",
|
||||
)
|
||||
|
||||
val levels = setOf("debug", "info", "warn", "error")
|
||||
|
||||
fun redact(value: String): String = value
|
||||
.replace(bearer) { match -> match.groupValues[1] + "***" }
|
||||
.replace(authorization) { match -> match.groupValues[1] + "***" }
|
||||
.replace(querySecret) { match -> match.groupValues[1] + "***" }
|
||||
.replace(jsonSecret) { match -> match.groupValues[1] + "***" + match.groupValues[2] }
|
||||
|
||||
fun normalizeLevel(value: String): String? =
|
||||
value.trim().lowercase().takeIf { it in levels }
|
||||
|
||||
fun deviceId(value: String): String = clean(value, MAX_DEVICE_ID).ifBlank { "unknown" }
|
||||
|
||||
fun platform(value: String): String = clean(value, MAX_PLATFORM).ifBlank { "unknown" }
|
||||
|
||||
fun appVersion(value: String): String = clean(value, MAX_APP_VERSION).ifBlank { "unknown" }
|
||||
|
||||
fun sessionId(value: String?): String? =
|
||||
value?.let { clean(it, MAX_SESSION_ID) }?.takeIf { it.isNotBlank() }
|
||||
|
||||
fun tag(value: String): String = clean(value, MAX_TAG).ifBlank { "app" }
|
||||
|
||||
fun message(value: String): String = clean(value, MAX_MESSAGE).ifBlank { "(empty)" }
|
||||
|
||||
fun detail(value: String?): String? =
|
||||
value?.let { clean(it, MAX_DETAIL) }?.takeIf { it.isNotBlank() }
|
||||
|
||||
private fun clean(value: String, max: Int): String =
|
||||
redact(value).trim().let { if (it.length > max) it.take(max) else it }
|
||||
}
|
||||
|
||||
/**
|
||||
* Client diagnostics storage. A batch is written in one transaction so a broken
|
||||
* request cannot leave half a log behind, and retention runs on the server clock.
|
||||
*/
|
||||
object ClientLogRepository {
|
||||
|
||||
private const val MAX_CLOCK_SKEW_MILLIS = 5 * 60_000L
|
||||
private const val MAX_ENTRY_AGE_MILLIS = 90L * 24 * 60 * 60_000
|
||||
|
||||
fun insert(userName: String, batch: ClientLogBatchDto): Int {
|
||||
val now = Instant.now()
|
||||
val receivedAt = now.toString()
|
||||
val deviceId = ClientLogSanitizer.deviceId(batch.deviceId)
|
||||
val platform = ClientLogSanitizer.platform(batch.platform)
|
||||
val appVersion = ClientLogSanitizer.appVersion(batch.appVersion)
|
||||
return transaction {
|
||||
var inserted = 0
|
||||
batch.entries.forEach { entry ->
|
||||
val level = ClientLogSanitizer.normalizeLevel(entry.level) ?: return@forEach
|
||||
val millis = entry.timestamp
|
||||
.takeIf { it > 0 }
|
||||
?.coerceIn(
|
||||
now.toEpochMilli() - MAX_ENTRY_AGE_MILLIS,
|
||||
now.toEpochMilli() + MAX_CLOCK_SKEW_MILLIS,
|
||||
)
|
||||
?: now.toEpochMilli()
|
||||
ClientLogsTable.insert {
|
||||
it[ClientLogsTable.userName] = userName
|
||||
it[ClientLogsTable.deviceId] = deviceId
|
||||
it[ClientLogsTable.platform] = platform
|
||||
it[ClientLogsTable.appVersion] = appVersion
|
||||
it[sessionId] = ClientLogSanitizer.sessionId(entry.sessionId)
|
||||
it[ClientLogsTable.level] = level
|
||||
it[tag] = ClientLogSanitizer.tag(entry.tag)
|
||||
it[message] = ClientLogSanitizer.message(entry.message)
|
||||
it[detail] = ClientLogSanitizer.detail(entry.detail)
|
||||
it[createdAt] = Instant.ofEpochMilli(millis).toString()
|
||||
it[ClientLogsTable.receivedAt] = receivedAt
|
||||
}
|
||||
inserted += 1
|
||||
}
|
||||
inserted
|
||||
}
|
||||
}
|
||||
|
||||
fun list(
|
||||
userName: String,
|
||||
level: String? = null,
|
||||
deviceId: String? = null,
|
||||
beforeId: Long? = null,
|
||||
sinceMillis: Long? = null,
|
||||
limit: Int = 200,
|
||||
): List<ClientLogRecord> {
|
||||
var condition: Op<Boolean> = ClientLogsTable.userName eq userName
|
||||
level?.let { condition = condition and (ClientLogsTable.level eq it) }
|
||||
deviceId?.let { condition = condition and (ClientLogsTable.deviceId eq it) }
|
||||
beforeId?.let { condition = condition and (ClientLogsTable.id less it) }
|
||||
sinceMillis?.let {
|
||||
val since = Instant.ofEpochMilli(it).toString()
|
||||
condition = condition and (ClientLogsTable.receivedAt greaterEq since)
|
||||
}
|
||||
return transaction {
|
||||
ClientLogsTable.selectAll()
|
||||
.where(condition)
|
||||
.orderBy(ClientLogsTable.id to SortOrder.DESC)
|
||||
.limit(limit.coerceIn(1, 1_000))
|
||||
.map { it.toRecord() }
|
||||
}
|
||||
}
|
||||
|
||||
fun countSince(userName: String, since: Instant): Long = transaction {
|
||||
ClientLogsTable.selectAll()
|
||||
.where {
|
||||
(ClientLogsTable.userName eq userName) and
|
||||
(ClientLogsTable.receivedAt greaterEq since.toString())
|
||||
}
|
||||
.count()
|
||||
}
|
||||
|
||||
fun delete(userName: String): Int = transaction {
|
||||
ClientLogsTable.deleteWhere { ClientLogsTable.userName eq userName }
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes rows older than [retentionDays] and trims every user to
|
||||
* [maxRowsPerUser], newest first. Returns the number of deleted rows.
|
||||
*/
|
||||
fun prune(retentionDays: Int, maxRowsPerUser: Int): Int {
|
||||
val cutoff = Instant.now()
|
||||
.minus(retentionDays.coerceAtLeast(1).toLong(), ChronoUnit.DAYS)
|
||||
.toString()
|
||||
var deleted = transaction {
|
||||
ClientLogsTable.deleteWhere { ClientLogsTable.receivedAt less cutoff }
|
||||
}
|
||||
val users = transaction {
|
||||
ClientLogsTable.select(ClientLogsTable.userName)
|
||||
.withDistinct()
|
||||
.map { it[ClientLogsTable.userName] }
|
||||
}
|
||||
users.forEach { user -> deleted += trim(user, maxRowsPerUser) }
|
||||
return deleted
|
||||
}
|
||||
|
||||
private fun trim(userName: String, maxRows: Int): Int {
|
||||
if (maxRows <= 0) return delete(userName)
|
||||
return transaction {
|
||||
val cutoff = ClientLogsTable.select(ClientLogsTable.id)
|
||||
.where { ClientLogsTable.userName eq userName }
|
||||
.orderBy(ClientLogsTable.id to SortOrder.DESC)
|
||||
.limit(1).offset((maxRows - 1).toLong())
|
||||
.map { it[ClientLogsTable.id] }
|
||||
.firstOrNull() ?: return@transaction 0
|
||||
ClientLogsTable.deleteWhere {
|
||||
(ClientLogsTable.userName eq userName) and
|
||||
(ClientLogsTable.id less cutoff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun ResultRow.toRecord(): ClientLogRecord = ClientLogRecord(
|
||||
id = this[ClientLogsTable.id],
|
||||
deviceId = this[ClientLogsTable.deviceId],
|
||||
platform = this[ClientLogsTable.platform],
|
||||
appVersion = this[ClientLogsTable.appVersion],
|
||||
sessionId = this[ClientLogsTable.sessionId],
|
||||
level = this[ClientLogsTable.level],
|
||||
tag = this[ClientLogsTable.tag],
|
||||
message = this[ClientLogsTable.message],
|
||||
detail = this[ClientLogsTable.detail],
|
||||
createdAt = this[ClientLogsTable.createdAt],
|
||||
receivedAt = this[ClientLogsTable.receivedAt],
|
||||
)
|
||||
}
|
||||
@@ -163,3 +163,24 @@ object SongAffinityTable : Table("song_affinity") {
|
||||
|
||||
override val primaryKey = PrimaryKey(userName, songId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Batches of client diagnostics. `received_at` is the server clock and is what
|
||||
* retention uses; `created_at` is the device clock and is only for display.
|
||||
*/
|
||||
object ClientLogsTable : Table("client_logs") {
|
||||
val id = long("id").autoIncrement()
|
||||
val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE)
|
||||
val deviceId = varchar("device_id", 80)
|
||||
val platform = varchar("platform", 32)
|
||||
val appVersion = varchar("app_version", 40)
|
||||
val sessionId = varchar("session_id", 64).nullable()
|
||||
val level = varchar("level", 8)
|
||||
val tag = varchar("tag", 64)
|
||||
val message = text("message")
|
||||
val detail = text("detail").nullable()
|
||||
val createdAt = varchar("created_at", 40)
|
||||
val receivedAt = varchar("received_at", 40)
|
||||
|
||||
override val primaryKey = PrimaryKey(id)
|
||||
}
|
||||
Reference in new issue
Block a user