From 44b75c4d4ddfb578f03d39fdb0e826df81962220 Mon Sep 17 00:00:00 2001 From: TapTap Date: Fri, 20 Nov 2026 13:43:05 +0000 Subject: [PATCH] feat: accept, store and expose client diagnostics - V11 client_logs table with per-user storage and retention/pruning - POST/GET/DELETE /logs with batch validation, rate limit and admin access - redact bearer tokens, Authorization headers and query/JSON secrets twice - add X-Request-Id to responses and include it in the server log pattern - report client error entries immediately in the server log - scripts/logs.sh for source-side retrieval and clearing - chart knobs CLIENT_LOG_RETENTION_DAYS and CLIENT_LOG_MAX_ROWS_PER_USER --- README.md | 43 +++- build.gradle.kts | 1 + charts/audiomanager/Chart.yaml | 2 +- charts/audiomanager/templates/deployment.yaml | 4 + charts/audiomanager/values.yaml | 6 + docs/design/mood-first.md | 13 +- scripts/logs.sh | 177 +++++++++++++ scripts/smoke.sh | 23 ++ .../win/taptap/musicmanager/Application.kt | 34 ++- .../win/taptap/musicmanager/api/ApiModels.kt | 45 ++++ .../win/taptap/musicmanager/api/ApiRoutes.kt | 3 + .../win/taptap/musicmanager/api/LogApi.kt | 142 +++++++++++ .../taptap/musicmanager/config/AppConfig.kt | 18 ++ .../musicmanager/db/ClientLogRepository.kt | 234 ++++++++++++++++++ .../win/taptap/musicmanager/db/Tables.kt | 21 ++ .../db/migration/V11__client_logs.sql | 24 ++ src/main/resources/logback.xml | 2 +- .../taptap/musicmanager/api/ApiRoutesTest.kt | 84 +++++++ .../db/ClientLogRepositoryTest.kt | 110 ++++++++ 19 files changed, 981 insertions(+), 5 deletions(-) create mode 100755 scripts/logs.sh create mode 100644 src/main/kotlin/win/taptap/musicmanager/api/LogApi.kt create mode 100644 src/main/kotlin/win/taptap/musicmanager/db/ClientLogRepository.kt create mode 100644 src/main/resources/db/migration/V11__client_logs.sql create mode 100644 src/test/kotlin/win/taptap/musicmanager/db/ClientLogRepositoryTest.kt diff --git a/README.md b/README.md index 71974da..37eafe9 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ Everything is environment variables: | `DEEZER_API_BASE` | `https://api.deezer.com` | Public Deezer API base | | `LOUDNESS_ENABLED` | `true` | Measure each file and expose a per-track gain | | `LOUDNESS_TARGET_LUFS` | `-16` | Integrated loudness target (`-30` to `-5`) | +| `CLIENT_LOG_RETENTION_DAYS` | `14` | Days uploaded client diagnostics are kept | +| `CLIENT_LOG_MAX_ROWS_PER_USER` | `20000` | Newest diagnostic rows kept per user | ### `users.yaml` @@ -114,6 +116,37 @@ Passwords live only in this file (use a Vault-managed Secret in Kubernetes). `admin` is only needed for the maintenance scan endpoints. Old `folders:` keys are accepted and ignored. +## Client diagnostics + +The apps keep a small local ring buffer (500 entries) of warnings, errors and +crashes. Warnings and errors are flushed immediately, so the line that explains +a crash survives it; info entries are batched. When signed in, the client uploads +up to 100 entries per minute to `POST /api/v1/logs`, tagged with a random install +id, platform, app version and per-run session id. + +Privacy and safety: + +- `Authorization` headers, `Bearer` tokens, `token=`/`password=`/`arl=` query + parameters and JSON secret fields are replaced with `***` on the client + **and** again on the server. +- The server stores rows per user; a user can only read their own logs. An + administrator can pass `user=`. +- Retention defaults to 14 days and 20,000 rows per user; both are configurable. +- A rejected batch (HTTP 400/413/422) is dropped so it cannot block later logs. + +Inspect them with the helper: + +```bash +scripts/logs.sh --server https://audiomanager.example.com --user alice \ + --level warn --limit 100 +# administrators can read another account +scripts/logs.sh --server ... --user admin --target alice +# clear after investigating +scripts/logs.sh --server ... --user alice --clear +``` + +The password is read from `LOGS_PASSWORD` or prompted; it is never stored. + ## Native API All endpoints are under `/api/v1` and require a bearer token, except login. @@ -126,10 +159,18 @@ Media URLs may use `?token=` because HTML audio and images cannot send headers. | `POST` | `/auth/login` | `{username,password,deviceName,platform}` → token | | `POST` | `/auth/logout` | Revoke the current token | | `GET` | `/me` | Current user | -| `GET` | `/info` | Default source, Deezer quality and loudness settings | +| `GET` | `/info` | Default source, Deezer quality, loudness and log retention | | `GET` | `/auth/devices` | List device tokens | | `DELETE` | `/auth/devices/{id}` | Revoke a device | +### Diagnostics + +| Method | Path | Purpose | +|---|---|---| +| `POST` | `/logs` | Upload a client diagnostics batch | +| `GET` | `/logs` | List diagnostics (`level`, `device`, `since`, `before`, `limit`; admins may add `user`) | +| `DELETE` | `/logs` | Delete the current user's (or, for admins, a named user's) diagnostics | + ### Moods | Method | Path | Purpose | diff --git a/build.gradle.kts b/build.gradle.kts index e052e2b..78c2d9e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -36,6 +36,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm:3.1.2") implementation("io.ktor:ktor-server-auto-head-response-jvm:3.1.2") implementation("io.ktor:ktor-server-cors-jvm:3.1.2") + implementation("io.ktor:ktor-server-call-id-jvm:3.1.2") // Database implementation("org.jetbrains.exposed:exposed-core:0.60.0") diff --git a/charts/audiomanager/Chart.yaml b/charts/audiomanager/Chart.yaml index 8f187c6..12f18f7 100644 --- a/charts/audiomanager/Chart.yaml +++ b/charts/audiomanager/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: audiomanager description: Mood-first self-hosted music server type: application -version: 0.4.1 +version: 0.5.0 appVersion: "0.1.0" diff --git a/charts/audiomanager/templates/deployment.yaml b/charts/audiomanager/templates/deployment.yaml index a067fa0..7060935 100644 --- a/charts/audiomanager/templates/deployment.yaml +++ b/charts/audiomanager/templates/deployment.yaml @@ -65,6 +65,10 @@ spec: value: {{ .Values.loudness.enabled | quote }} - name: LOUDNESS_TARGET_LUFS value: {{ .Values.loudness.targetLufs | quote }} + - name: CLIENT_LOG_RETENTION_DAYS + value: {{ .Values.clientLogs.retentionDays | quote }} + - name: CLIENT_LOG_MAX_ROWS_PER_USER + value: {{ .Values.clientLogs.maxRowsPerUser | quote }} {{- if .Values.deezer.existingSecret }} - name: DEEZER_ARL valueFrom: diff --git a/charts/audiomanager/values.yaml b/charts/audiomanager/values.yaml index 344f6bf..b6b8e36 100644 --- a/charts/audiomanager/values.yaml +++ b/charts/audiomanager/values.yaml @@ -47,6 +47,12 @@ loudness: # -16 LUFS is the ReplayGain-style target; -14 matches most streaming services. targetLufs: -16 +# Client diagnostics uploaded by the apps. The server redacts credentials again +# and keeps only a short window so bug reports do not grow the database. +clientLogs: + retentionDays: 14 + maxRowsPerUser: 20000 + # The media root, database, cover cache and download temp dir all live here. data: persistence: diff --git a/docs/design/mood-first.md b/docs/design/mood-first.md index 3973900..b522212 100644 --- a/docs/design/mood-first.md +++ b/docs/design/mood-first.md @@ -750,7 +750,18 @@ Phases 1-8 are implemented in both repositories. additions in sync on unmetered networks, three at a time. - **Sleep timer** adds "Finish this track"; the full-screen player shows large artwork, transport, add-to-mood, queue and Up next. -- Tests: 21 client tests; desktop, Android debug and Wasm compile. +- **Diagnostics (V11)**: the client keeps 500 local info/warn/error entries and + uploads up to 100 per minute to `POST /logs`, tagged with a random install id, + platform, app version and per-process session id. Warnings and errors flush + immediately so a crash keeps its cause. `Authorization`, `Bearer`, `token=`, + `password=` and `arl=` values are redacted client-side and again server-side. + Rows are per user, admins may read another user with `?user=`, and retention is + `CLIENT_LOG_RETENTION_DAYS` (14) plus `CLIENT_LOG_MAX_ROWS_PER_USER` (20000). + Uncaught exceptions are stored on Android and desktop and uploaded on the next + launch. Every response carries `X-Request-Id`, which the client includes in + server-error diagnostics so a report can be matched to the server log. +- Tests: 26 client tests; desktop, Android debug and Wasm compile. + ### Known follow-ups diff --git a/scripts/logs.sh b/scripts/logs.sh new file mode 100755 index 0000000..d50375a --- /dev/null +++ b/scripts/logs.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# +# Fetch and print client diagnostics stored by the server. +# +# The password is read from LOGS_PASSWORD or prompted, and is only used for the +# login request. The token is revoked before the script exits. +# +# Usage: +# scripts/logs.sh --server https://music.example.com --user alice +# scripts/logs.sh --server ... --user admin --target bob --level error +# scripts/logs.sh --server ... --user alice --since 7d --limit 500 +# scripts/logs.sh --server ... --user alice --clear +set -euo pipefail + +SERVER="" +USER_NAME="" +PASSWORD="${LOGS_PASSWORD:-}" +TARGET="" +LEVEL="" +DEVICE="" +LIMIT="200" +SINCE="" +CLEAR=0 +RAW=0 + +usage() { + cat <<'EOF' +Usage: scripts/logs.sh --server URL --user NAME [options] + +Options: + --password PW Password (otherwise LOGS_PASSWORD or a prompt) + --target NAME Read another user's logs (administrators only) + --level LEVEL debug, info, warn or error + --device ID Only this install id + --since VALUE Epoch milliseconds or a window such as 7d / 12h + --limit N Rows to fetch (1..1000, default 200) + --clear Delete the selected user's logs instead of printing them + --raw Print the JSON response unchanged + -h, --help Show this help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --server) SERVER="${2:?--server needs a value}"; shift 2 ;; + --user) USER_NAME="${2:?--user needs a value}"; shift 2 ;; + --password) PASSWORD="${2:?--password needs a value}"; shift 2 ;; + --target) TARGET="${2:?--target needs a value}"; shift 2 ;; + --level) LEVEL="${2:?--level needs a value}"; shift 2 ;; + --device) DEVICE="${2:?--device needs a value}"; shift 2 ;; + --since) SINCE="${2:?--since needs a value}"; shift 2 ;; + --limit) LIMIT="${2:?--limit needs a value}"; shift 2 ;; + --clear) CLEAR=1; shift ;; + --raw) RAW=1; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ -z "$SERVER" || -z "$USER_NAME" ]]; then + usage >&2 + exit 2 +fi +SERVER="${SERVER%/}" + +if [[ -z "$PASSWORD" ]]; then + read -rsp "Password for $USER_NAME: " PASSWORD + echo +fi + +since_millis() { + local value="$1" + if [[ "$value" =~ ^([0-9]+)([dh])$ ]]; then + python3 - "$value" <<'PY' +import re, sys, time +value = sys.argv[1] +amount = int(re.match(r"([0-9]+)", value).group(1)) +unit = value[-1] +seconds = amount * (86400 if unit == "d" else 3600) +print(int((time.time() - seconds) * 1000)) +PY + else + printf '%s' "$value" + fi +} + +# Keep the password out of argv: it is piped to the JSON encoder as stdin. +login_body="$(printf '%s\n%s\n' "$USER_NAME" "$PASSWORD" | python3 -c ' +import json, sys +user = sys.stdin.readline().rstrip("\n") +password = sys.stdin.readline().rstrip("\n") +print(json.dumps({ + "username": user, + "password": password, + "deviceName": "diagnostics-cli", + "platform": "cli", +})) +')" + +login_response="$(printf '%s' "$login_body" | curl -sS -w '\n%{http_code}' -X POST \ + "$SERVER/api/v1/auth/login" \ + -H 'Content-Type: application/json' \ + --data-binary @-)" +login_status="$(printf '%s' "$login_response" | tail -1)" +login_body="$(printf '%s' "$login_response" | sed '$d')" +if [[ "$login_status" != "200" ]]; then + echo "Login failed (HTTP $login_status): $login_body" >&2 + exit 1 +fi +TOKEN="$(printf '%s' "$login_body" | python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')" + +# The token stays in a 0600 curl config instead of the process list. +AUTH_CONFIG="$(mktemp "${TMPDIR:-/tmp}/audiomanager-logs.XXXXXX")" +PAYLOAD_FILE="" +chmod 600 "$AUTH_CONFIG" +printf 'header = "Authorization: Bearer %s"\n' "$TOKEN" > "$AUTH_CONFIG" + +cleanup() { + curl -sS -o /dev/null -X POST "$SERVER/api/v1/auth/logout" \ + -K "$AUTH_CONFIG" 2>/dev/null || true + rm -f "$AUTH_CONFIG" "$PAYLOAD_FILE" +} +trap cleanup EXIT + +query=() +[[ -n "$LEVEL" ]] && query+=(--data-urlencode "level=$LEVEL") +[[ -n "$DEVICE" ]] && query+=(--data-urlencode "device=$DEVICE") +[[ -n "$TARGET" ]] && query+=(--data-urlencode "user=$TARGET") +[[ -n "$SINCE" ]] && query+=(--data-urlencode "since=$(since_millis "$SINCE")") + +if [[ "$CLEAR" -eq 1 ]]; then + response="$(curl -sS -G -X DELETE "$SERVER/api/v1/logs" \ + -K "$AUTH_CONFIG" "${query[@]}")" + printf '%s' "$response" | python3 -c 'import json,sys; print("deleted", json.load(sys.stdin)["deleted"], "entries")' + exit 0 +fi + +if [[ "$LIMIT" != "200" ]]; then + query+=(--data-urlencode "limit=$LIMIT") +fi + +response="$(curl -sS -G "$SERVER/api/v1/logs" \ + -K "$AUTH_CONFIG" "${query[@]}")" + +PAYLOAD_FILE="$(mktemp "${TMPDIR:-/tmp}/audiomanager-logs-payload.XXXXXX")" +printf '%s' "$response" > "$PAYLOAD_FILE" + +python3 - "$PAYLOAD_FILE" "$RAW" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as handle: + data = json.load(handle) +if sys.argv[2] == "1": + print(json.dumps(data, indent=2)) + raise SystemExit(0) + +logs = data.get("logs", []) +if not logs: + print("No diagnostics stored for this user.") + raise SystemExit(0) + +print(f"{len(logs)} entr{'y' if len(logs) == 1 else 'ies'} (newest first)") +print() +for row in logs: + when = row.get("createdAt") or row.get("receivedAt") or "?" + device = f"{row.get('platform', '?')} {row.get('appVersion', '?')} {row.get('deviceId', '?')}" + session = row.get("sessionId") + suffix = f" session={session}" if session else "" + print(f"{when} {row.get('level', '?').upper():5} {row.get('tag', '?')} [{device}]{suffix}") + print(f" {row.get('message', '')}") + detail = row.get("detail") + if detail: + for line in detail.splitlines(): + print(f" | {line}") + print() +PY diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 19fb8d8..95d1ce8 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -498,6 +498,29 @@ fi expect "scan status" "200" "GET" "/scan" expect "start scan" "202" "POST" "/scan" +log "" +log "Diagnostics" +expect "upload client logs" "200" "POST" "/logs" \ + '{"deviceId":"smoke-dev","platform":"desktop","appVersion":"0.4.3","entries":[{"timestamp":0,"level":"error","tag":"smoke","message":"failed with Bearer smoke-secret","detail":"https://music/stream?token=abc123"}]}' +expect_eq "client log accepted" "1" "$(printf '%s' "$HTTP_BODY" | json_field '["accepted"]')" +expect "list client logs" "200" "GET" "/logs" +LOGS_RESULT="$(printf '%s' "$HTTP_BODY" | python3 -c ' +import json, sys +logs = json.load(sys.stdin)["logs"] +text = json.dumps(logs) +ok = len(logs) == 1 and logs[0]["level"] == "error" and "smoke-secret" not in text and "abc123" not in text +print("ok" if ok else "bad") +')" +expect_eq "client logs are stored and redacted" "ok" "$LOGS_RESULT" +LOGS_OUTPUT="$(LOGS_PASSWORD="$PASS" "$ROOT/scripts/logs.sh" --server "$BASE" --user "$USER" --limit 5 2>&1)" || true +if printf '%s' "$LOGS_OUTPUT" | grep -q 'failed with Bearer \*\*\*'; then + pass "logs.sh prints redacted diagnostics" +else + fail "logs.sh prints redacted diagnostics" "${LOGS_OUTPUT:0:240}" +fi +expect "clear client logs" "200" "DELETE" "/logs" +expect_eq "client logs deleted" "1" "$(printf '%s' "$HTTP_BODY" | json_field '["deleted"]')" + log "" log "Cleanup" expect "delete mood" "204" "DELETE" "/moods/$MOOD_ID" diff --git a/src/main/kotlin/win/taptap/musicmanager/Application.kt b/src/main/kotlin/win/taptap/musicmanager/Application.kt index f730557..05e5d63 100644 --- a/src/main/kotlin/win/taptap/musicmanager/Application.kt +++ b/src/main/kotlin/win/taptap/musicmanager/Application.kt @@ -9,6 +9,8 @@ import io.ktor.server.application.install import io.ktor.server.engine.embeddedServer import io.ktor.server.netty.Netty import io.ktor.server.plugins.autohead.AutoHeadResponse +import io.ktor.server.plugins.callid.CallId +import io.ktor.server.plugins.callid.callIdMdc import io.ktor.server.plugins.calllogging.CallLogging import io.ktor.server.plugins.contentnegotiation.ContentNegotiation import io.ktor.server.plugins.cors.routing.CORS @@ -20,6 +22,7 @@ import io.ktor.server.response.respondText import io.ktor.server.routing.get import io.ktor.server.routing.routing import java.nio.file.Files +import java.util.UUID import java.util.concurrent.TimeUnit import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -38,6 +41,7 @@ import win.taptap.musicmanager.api.ApiException import win.taptap.musicmanager.api.apiRoutes import win.taptap.musicmanager.config.AppConfig import win.taptap.musicmanager.config.UsersConfig +import win.taptap.musicmanager.db.ClientLogRepository import win.taptap.musicmanager.db.ConfigRepository import win.taptap.musicmanager.db.DatabaseFactory import win.taptap.musicmanager.db.MoodRepository @@ -184,6 +188,25 @@ fun main() { } } + // Client diagnostics are kept for a short window so a bug can be + // investigated without letting the table grow without bound. + applicationScope.launch(Dispatchers.IO) { + delay(TimeUnit.MINUTES.toMillis(1)) + while (isActive) { + runCatching { + ClientLogRepository.prune( + config.clientLogRetentionDays, + config.clientLogMaxRowsPerUser, + ) + } + .onSuccess { count -> + if (count > 0) log.info("Pruned {} client diagnostics rows", count) + } + .onFailure { log.warn("Client diagnostics prune failed", it) } + delay(TimeUnit.HOURS.toMillis(6)) + } + } + // Delete files that no mood, recommendation or open session references. applicationScope.launch(Dispatchers.IO) { delay(TimeUnit.MINUTES.toMillis(10)) @@ -221,7 +244,15 @@ fun main() { } install(PartialContent) install(AutoHeadResponse) - install(CallLogging) + // A short id on every response makes server log lines and client + // diagnostics meet in the middle. + install(CallId) { + generate { UUID.randomUUID().toString().take(8) } + replyToHeader("X-Request-Id") + } + install(CallLogging) { + callIdMdc("requestId") + } install(StatusPages) { exception { call, cause -> call.respond( @@ -271,6 +302,7 @@ fun main() { deezerQuality = config.deezerQuality, loudnessEnabled = config.loudnessEnabled && loudnessAnalyzer != null, loudnessTargetLufs = config.loudnessTargetLufs, + clientLogRetentionDays = config.clientLogRetentionDays, startScan = { scanner.start(applicationScope) }, scanStatus = { scanner.status() }, ) diff --git a/src/main/kotlin/win/taptap/musicmanager/api/ApiModels.kt b/src/main/kotlin/win/taptap/musicmanager/api/ApiModels.kt index 48e17b4..b842114 100644 --- a/src/main/kotlin/win/taptap/musicmanager/api/ApiModels.kt +++ b/src/main/kotlin/win/taptap/musicmanager/api/ApiModels.kt @@ -44,6 +44,49 @@ data class DeviceDto( val current: Boolean, ) +@Serializable +data class ClientLogEntryDto( + /** Device clock in epoch milliseconds; informational, the server clamps it. */ + val timestamp: Long = 0, + val level: String, + val tag: String, + val message: String, + val detail: String? = null, + val sessionId: String? = null, +) + +@Serializable +data class ClientLogBatchDto( + val deviceId: String = "unknown", + val platform: String = "unknown", + val appVersion: String = "unknown", + val entries: List = emptyList(), +) + +@Serializable +data class ClientLogDto( + val id: Long, + val deviceId: String, + val platform: String, + val appVersion: String, + val sessionId: String? = null, + val level: String, + val tag: String, + val message: String, + val detail: String? = null, + val createdAt: String, + val receivedAt: String, +) + +@Serializable +data class ClientLogsDto(val logs: List) + +@Serializable +data class ClientLogsAcceptedDto(val accepted: Int) + +@Serializable +data class ClientLogsDeletedDto(val deleted: Int) + @Serializable data class ServerInfoDto( val defaultSource: String, @@ -51,6 +94,8 @@ data class ServerInfoDto( val deezerQuality: Int, val loudnessEnabled: Boolean, val loudnessTargetLufs: Double, + /** How long the server keeps uploaded client diagnostics. */ + val clientLogRetentionDays: Int, ) @Serializable diff --git a/src/main/kotlin/win/taptap/musicmanager/api/ApiRoutes.kt b/src/main/kotlin/win/taptap/musicmanager/api/ApiRoutes.kt index 3116744..3758d6f 100644 --- a/src/main/kotlin/win/taptap/musicmanager/api/ApiRoutes.kt +++ b/src/main/kotlin/win/taptap/musicmanager/api/ApiRoutes.kt @@ -40,6 +40,7 @@ fun Route.apiRoutes( deezerQuality: Int, loudnessEnabled: Boolean, loudnessTargetLufs: Double, + clientLogRetentionDays: Int, startScan: () -> Boolean, scanStatus: () -> ScanStatus, ) { @@ -89,6 +90,7 @@ fun Route.apiRoutes( deezerQuality = deezerQuality, loudnessEnabled = loudnessEnabled, loudnessTargetLufs = loudnessTargetLufs, + clientLogRetentionDays = clientLogRetentionDays, ), ) } @@ -151,6 +153,7 @@ fun Route.apiRoutes( defaultSource = defaultSource, ) sessionRoutes(users, radioService, gc) + logRoutes(users) } } diff --git a/src/main/kotlin/win/taptap/musicmanager/api/LogApi.kt b/src/main/kotlin/win/taptap/musicmanager/api/LogApi.kt new file mode 100644 index 0000000..c686c97 --- /dev/null +++ b/src/main/kotlin/win/taptap/musicmanager/api/LogApi.kt @@ -0,0 +1,142 @@ +package win.taptap.musicmanager.api + +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.contentLength +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import java.time.Instant +import java.time.temporal.ChronoUnit +import org.slf4j.LoggerFactory +import win.taptap.musicmanager.config.UserAccount +import win.taptap.musicmanager.config.UsersConfig +import win.taptap.musicmanager.db.ClientLogRepository +import win.taptap.musicmanager.db.ClientLogSanitizer +import win.taptap.musicmanager.db.toDto + +private val log = LoggerFactory.getLogger("win.taptap.musicmanager.api.Logs") + +/** A diagnostic batch is small; anything bigger is a bug or abuse. */ +private const val MAX_BODY_BYTES = 2L * 1024 * 1024 + +/** Keep the insert transaction short and the per-request cost predictable. */ +private const val MAX_ENTRIES_PER_BATCH = 200 + +/** Per-user ingest ceiling, measured over the last hour. */ +private const val MAX_ROWS_PER_HOUR = 20_000 + +/** + * Client diagnostics endpoints. + * + * The app uploads warnings/errors and a few context entries in batches. Each + * row belongs to the authenticated user (or to the user an administrator names), + * so one account can never read another account's logs. The server redacts + * credentials again before storing anything. + */ +fun Route.logRoutes(users: UsersConfig) { + + post("/logs") { + val user = call.requireApiUser(users) + val length = call.request.contentLength() + if (length != null && length > MAX_BODY_BYTES) { + throw ApiException( + HttpStatusCode.PayloadTooLarge, + "logs_too_large", + "A log batch may not exceed ${MAX_BODY_BYTES / 1024} KiB", + ) + } + val batch = call.receiveJson() + if (batch.entries.isEmpty()) badRequest("A log batch needs at least one entry") + if (batch.entries.size > MAX_ENTRIES_PER_BATCH) { + badRequest("A log batch may contain at most $MAX_ENTRIES_PER_BATCH entries") + } + val payloadBytes = batch.entries.sumOf { entry -> + entry.message.length + (entry.detail?.length ?: 0) + entry.tag.length + } + if (payloadBytes > MAX_BODY_BYTES) { + throw ApiException( + HttpStatusCode.PayloadTooLarge, + "logs_too_large", + "A log batch may not exceed ${MAX_BODY_BYTES / 1024} KiB", + ) + } + batch.entries.forEach { entry -> + if (ClientLogSanitizer.normalizeLevel(entry.level) == null) { + badRequest("Invalid log level: ${entry.level.take(16)}") + } + } + + val since = Instant.now().minus(1, ChronoUnit.HOURS) + if (ClientLogRepository.countSince(user.name, since) >= MAX_ROWS_PER_HOUR) { + throw ApiException( + HttpStatusCode.TooManyRequests, + "logs_rate_limited", + "Too many diagnostic entries in the last hour", + ) + } + + val accepted = ClientLogRepository.insert(user.name, batch) + // Errors should show up in the server log immediately, not only when + // someone remembers to query the diagnostics endpoint. + batch.entries + .filter { ClientLogSanitizer.normalizeLevel(it.level) == "error" } + .take(3) + .forEach { entry -> + log.warn( + "Client error [{}] {}: {}", + user.name, + ClientLogSanitizer.tag(entry.tag), + ClientLogSanitizer.message(entry.message).take(300), + ) + } + call.respond(ClientLogsAcceptedDto(accepted)) + } + + get("/logs") { + val user = call.requireApiUser(users) + val target = call.logTargetUser(users, user) + val level = call.request.queryParameters["level"] + ?.takeIf { it.isNotBlank() } + ?.let { + ClientLogSanitizer.normalizeLevel(it) + ?: badRequest("level must be one of ${ClientLogSanitizer.levels.sorted().joinToString()}") + } + val deviceId = call.request.queryParameters["device"]?.takeIf { it.isNotBlank() } + val beforeId = call.request.queryParameters["before"]?.let { raw -> + raw.toLongOrNull()?.takeIf { it > 0 } ?: badRequest("before must be a positive row id") + } + val sinceMillis = call.request.queryParameters["since"]?.let { raw -> + raw.toLongOrNull()?.takeIf { it > 0 } ?: badRequest("since must be epoch milliseconds") + } + val limit = call.request.queryParameters["limit"]?.let { raw -> + raw.toIntOrNull()?.takeIf { it in 1..1_000 } ?: badRequest("limit must be between 1 and 1000") + } ?: 200 + + val logs = ClientLogRepository.list( + userName = target, + level = level, + deviceId = deviceId, + beforeId = beforeId, + sinceMillis = sinceMillis, + limit = limit, + ) + call.respond(ClientLogsDto(logs.map { it.toDto() })) + } + + delete("/logs") { + val user = call.requireApiUser(users) + val target = call.logTargetUser(users, user) + val deleted = ClientLogRepository.delete(target) + call.respond(ClientLogsDeletedDto(deleted)) + } +} + +private fun ApplicationCall.logTargetUser(users: UsersConfig, current: UserAccount): String { + val requested = request.queryParameters["user"]?.takeIf { it.isNotBlank() } + if (requested == null || requested == current.name) return current.name + if (!current.admin) forbidden("Only administrators can read another user's logs") + return users.byName(requested)?.name ?: notFound("User") +} diff --git a/src/main/kotlin/win/taptap/musicmanager/config/AppConfig.kt b/src/main/kotlin/win/taptap/musicmanager/config/AppConfig.kt index be6becd..8412cb1 100644 --- a/src/main/kotlin/win/taptap/musicmanager/config/AppConfig.kt +++ b/src/main/kotlin/win/taptap/musicmanager/config/AppConfig.kt @@ -26,6 +26,10 @@ data class AppConfig( val loudnessEnabled: Boolean = true, /** Integrated loudness target; -16 LUFS is the ReplayGain-style value. */ val loudnessTargetLufs: Double = -16.0, + /** Client diagnostics: rows older than this are pruned. */ + val clientLogRetentionDays: Int = 14, + /** Client diagnostics: newest rows kept per user. */ + val clientLogMaxRowsPerUser: Int = 20_000, val mediaDir: Path = dataDir.resolve("media"), val downloadDir: Path = dataDir.resolve("downloads"), ) { @@ -62,6 +66,18 @@ data class AppConfig( require(loudnessTargetLufs in -30.0..-5.0) { "LOUDNESS_TARGET_LUFS must be between -30 and -5, got: $loudnessTargetLufs" } + val rawRetention = value("CLIENT_LOG_RETENTION_DAYS", "14") + val clientLogRetentionDays = rawRetention.toIntOrNull() + ?: throw IllegalArgumentException("CLIENT_LOG_RETENTION_DAYS must be an integer") + require(clientLogRetentionDays in 1..365) { + "CLIENT_LOG_RETENTION_DAYS must be between 1 and 365, got: $clientLogRetentionDays" + } + val rawLogRows = value("CLIENT_LOG_MAX_ROWS_PER_USER", "20000") + val clientLogMaxRowsPerUser = rawLogRows.toIntOrNull() + ?: throw IllegalArgumentException("CLIENT_LOG_MAX_ROWS_PER_USER must be an integer") + require(clientLogMaxRowsPerUser in 100..1_000_000) { + "CLIENT_LOG_MAX_ROWS_PER_USER must be between 100 and 1000000, got: $clientLogMaxRowsPerUser" + } return AppConfig( port = port, @@ -76,6 +92,8 @@ data class AppConfig( deezerApiBase = value("DEEZER_API_BASE", "https://api.deezer.com"), loudnessEnabled = loudnessEnabled, loudnessTargetLufs = loudnessTargetLufs, + clientLogRetentionDays = clientLogRetentionDays, + clientLogMaxRowsPerUser = clientLogMaxRowsPerUser, mediaDir = Path.of(value("MEDIA_DIR", dataDir.resolve("media").toString())), downloadDir = Path.of(value("DOWNLOAD_DIR", dataDir.resolve("downloads").toString())), ) diff --git a/src/main/kotlin/win/taptap/musicmanager/db/ClientLogRepository.kt b/src/main/kotlin/win/taptap/musicmanager/db/ClientLogRepository.kt new file mode 100644 index 0000000..c6ca09c --- /dev/null +++ b/src/main/kotlin/win/taptap/musicmanager/db/ClientLogRepository.kt @@ -0,0 +1,234 @@ +package win.taptap.musicmanager.db + +import java.time.Instant +import java.time.temporal.ChronoUnit +import org.jetbrains.exposed.sql.Op +import org.jetbrains.exposed.sql.ResultRow +import org.jetbrains.exposed.sql.SortOrder +import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq +import org.jetbrains.exposed.sql.SqlExpressionBuilder.greaterEq +import org.jetbrains.exposed.sql.SqlExpressionBuilder.less +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.deleteWhere +import org.jetbrains.exposed.sql.insert +import org.jetbrains.exposed.sql.selectAll +import org.jetbrains.exposed.sql.transactions.transaction +import win.taptap.musicmanager.api.ClientLogBatchDto +import win.taptap.musicmanager.api.ClientLogDto + +data class ClientLogRecord( + val id: Long, + val deviceId: String, + val platform: String, + val appVersion: String, + val sessionId: String?, + val level: String, + val tag: String, + val message: String, + val detail: String?, + val createdAt: String, + val receivedAt: String, +) + +fun ClientLogRecord.toDto(): ClientLogDto = ClientLogDto( + id = id, + deviceId = deviceId, + platform = platform, + appVersion = appVersion, + sessionId = sessionId, + level = level, + tag = tag, + message = message, + detail = detail, + createdAt = createdAt, + receivedAt = receivedAt, +) + +/** + * Removes credentials before anything is persisted. The client redacts as well, + * but the server must not trust a buggy or older build: a token in a log line + * would give whoever reads the table access to the account. + */ +object ClientLogSanitizer { + + private const val MAX_MESSAGE = 4_000 + private const val MAX_DETAIL = 16_000 + private const val MAX_TAG = 64 + private const val MAX_DEVICE_ID = 80 + private const val MAX_PLATFORM = 32 + private const val MAX_APP_VERSION = 40 + private const val MAX_SESSION_ID = 64 + + private val bearer = Regex("""(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]{8,}""") + private val authorization = Regex("""(?i)\b(authorization\s*:\s*)[^\r\n]+""") + private val querySecret = Regex( + """(?i)(\b(?:token|access_token|password|passwd|arl|key)=)[^&\s"'<>]+""", + ) + private val jsonSecret = Regex( + """(?i)("(?:token|access_token|password|arl|key)"\s*:\s*")[^"]*(")""", + ) + + val levels = setOf("debug", "info", "warn", "error") + + fun redact(value: String): String = value + .replace(bearer) { match -> match.groupValues[1] + "***" } + .replace(authorization) { match -> match.groupValues[1] + "***" } + .replace(querySecret) { match -> match.groupValues[1] + "***" } + .replace(jsonSecret) { match -> match.groupValues[1] + "***" + match.groupValues[2] } + + fun normalizeLevel(value: String): String? = + value.trim().lowercase().takeIf { it in levels } + + fun deviceId(value: String): String = clean(value, MAX_DEVICE_ID).ifBlank { "unknown" } + + fun platform(value: String): String = clean(value, MAX_PLATFORM).ifBlank { "unknown" } + + fun appVersion(value: String): String = clean(value, MAX_APP_VERSION).ifBlank { "unknown" } + + fun sessionId(value: String?): String? = + value?.let { clean(it, MAX_SESSION_ID) }?.takeIf { it.isNotBlank() } + + fun tag(value: String): String = clean(value, MAX_TAG).ifBlank { "app" } + + fun message(value: String): String = clean(value, MAX_MESSAGE).ifBlank { "(empty)" } + + fun detail(value: String?): String? = + value?.let { clean(it, MAX_DETAIL) }?.takeIf { it.isNotBlank() } + + private fun clean(value: String, max: Int): String = + redact(value).trim().let { if (it.length > max) it.take(max) else it } +} + +/** + * Client diagnostics storage. A batch is written in one transaction so a broken + * request cannot leave half a log behind, and retention runs on the server clock. + */ +object ClientLogRepository { + + private const val MAX_CLOCK_SKEW_MILLIS = 5 * 60_000L + private const val MAX_ENTRY_AGE_MILLIS = 90L * 24 * 60 * 60_000 + + fun insert(userName: String, batch: ClientLogBatchDto): Int { + val now = Instant.now() + val receivedAt = now.toString() + val deviceId = ClientLogSanitizer.deviceId(batch.deviceId) + val platform = ClientLogSanitizer.platform(batch.platform) + val appVersion = ClientLogSanitizer.appVersion(batch.appVersion) + return transaction { + var inserted = 0 + batch.entries.forEach { entry -> + val level = ClientLogSanitizer.normalizeLevel(entry.level) ?: return@forEach + val millis = entry.timestamp + .takeIf { it > 0 } + ?.coerceIn( + now.toEpochMilli() - MAX_ENTRY_AGE_MILLIS, + now.toEpochMilli() + MAX_CLOCK_SKEW_MILLIS, + ) + ?: now.toEpochMilli() + ClientLogsTable.insert { + it[ClientLogsTable.userName] = userName + it[ClientLogsTable.deviceId] = deviceId + it[ClientLogsTable.platform] = platform + it[ClientLogsTable.appVersion] = appVersion + it[sessionId] = ClientLogSanitizer.sessionId(entry.sessionId) + it[ClientLogsTable.level] = level + it[tag] = ClientLogSanitizer.tag(entry.tag) + it[message] = ClientLogSanitizer.message(entry.message) + it[detail] = ClientLogSanitizer.detail(entry.detail) + it[createdAt] = Instant.ofEpochMilli(millis).toString() + it[ClientLogsTable.receivedAt] = receivedAt + } + inserted += 1 + } + inserted + } + } + + fun list( + userName: String, + level: String? = null, + deviceId: String? = null, + beforeId: Long? = null, + sinceMillis: Long? = null, + limit: Int = 200, + ): List { + var condition: Op = ClientLogsTable.userName eq userName + level?.let { condition = condition and (ClientLogsTable.level eq it) } + deviceId?.let { condition = condition and (ClientLogsTable.deviceId eq it) } + beforeId?.let { condition = condition and (ClientLogsTable.id less it) } + sinceMillis?.let { + val since = Instant.ofEpochMilli(it).toString() + condition = condition and (ClientLogsTable.receivedAt greaterEq since) + } + return transaction { + ClientLogsTable.selectAll() + .where(condition) + .orderBy(ClientLogsTable.id to SortOrder.DESC) + .limit(limit.coerceIn(1, 1_000)) + .map { it.toRecord() } + } + } + + fun countSince(userName: String, since: Instant): Long = transaction { + ClientLogsTable.selectAll() + .where { + (ClientLogsTable.userName eq userName) and + (ClientLogsTable.receivedAt greaterEq since.toString()) + } + .count() + } + + fun delete(userName: String): Int = transaction { + ClientLogsTable.deleteWhere { ClientLogsTable.userName eq userName } + } + + /** + * Deletes rows older than [retentionDays] and trims every user to + * [maxRowsPerUser], newest first. Returns the number of deleted rows. + */ + fun prune(retentionDays: Int, maxRowsPerUser: Int): Int { + val cutoff = Instant.now() + .minus(retentionDays.coerceAtLeast(1).toLong(), ChronoUnit.DAYS) + .toString() + var deleted = transaction { + ClientLogsTable.deleteWhere { ClientLogsTable.receivedAt less cutoff } + } + val users = transaction { + ClientLogsTable.select(ClientLogsTable.userName) + .withDistinct() + .map { it[ClientLogsTable.userName] } + } + users.forEach { user -> deleted += trim(user, maxRowsPerUser) } + return deleted + } + + private fun trim(userName: String, maxRows: Int): Int { + if (maxRows <= 0) return delete(userName) + return transaction { + val cutoff = ClientLogsTable.select(ClientLogsTable.id) + .where { ClientLogsTable.userName eq userName } + .orderBy(ClientLogsTable.id to SortOrder.DESC) + .limit(1).offset((maxRows - 1).toLong()) + .map { it[ClientLogsTable.id] } + .firstOrNull() ?: return@transaction 0 + ClientLogsTable.deleteWhere { + (ClientLogsTable.userName eq userName) and + (ClientLogsTable.id less cutoff) + } + } + } + + private fun ResultRow.toRecord(): ClientLogRecord = ClientLogRecord( + id = this[ClientLogsTable.id], + deviceId = this[ClientLogsTable.deviceId], + platform = this[ClientLogsTable.platform], + appVersion = this[ClientLogsTable.appVersion], + sessionId = this[ClientLogsTable.sessionId], + level = this[ClientLogsTable.level], + tag = this[ClientLogsTable.tag], + message = this[ClientLogsTable.message], + detail = this[ClientLogsTable.detail], + createdAt = this[ClientLogsTable.createdAt], + receivedAt = this[ClientLogsTable.receivedAt], + ) +} diff --git a/src/main/kotlin/win/taptap/musicmanager/db/Tables.kt b/src/main/kotlin/win/taptap/musicmanager/db/Tables.kt index 7ec60ca..a30f2a9 100644 --- a/src/main/kotlin/win/taptap/musicmanager/db/Tables.kt +++ b/src/main/kotlin/win/taptap/musicmanager/db/Tables.kt @@ -163,3 +163,24 @@ object SongAffinityTable : Table("song_affinity") { override val primaryKey = PrimaryKey(userName, songId) } + +/** + * Batches of client diagnostics. `received_at` is the server clock and is what + * retention uses; `created_at` is the device clock and is only for display. + */ +object ClientLogsTable : Table("client_logs") { + val id = long("id").autoIncrement() + val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE) + val deviceId = varchar("device_id", 80) + val platform = varchar("platform", 32) + val appVersion = varchar("app_version", 40) + val sessionId = varchar("session_id", 64).nullable() + val level = varchar("level", 8) + val tag = varchar("tag", 64) + val message = text("message") + val detail = text("detail").nullable() + val createdAt = varchar("created_at", 40) + val receivedAt = varchar("received_at", 40) + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/resources/db/migration/V11__client_logs.sql b/src/main/resources/db/migration/V11__client_logs.sql new file mode 100644 index 0000000..fbf1fda --- /dev/null +++ b/src/main/resources/db/migration/V11__client_logs.sql @@ -0,0 +1,24 @@ +-- Client diagnostics. The apps keep a small local ring buffer of warnings and +-- errors (plus an occasional info entry) and upload it in batches so a bug can +-- be investigated with the server-side request id and the client context. +-- +-- Rows are per user and pruned by retention days and a per-user row cap. The +-- client timestamp is informational; ordering uses the server-assigned id so a +-- device with a wrong clock cannot reorder the log. +CREATE TABLE client_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL REFERENCES users(name) ON DELETE CASCADE, + device_id TEXT NOT NULL, + platform TEXT NOT NULL, + app_version TEXT NOT NULL, + session_id TEXT, + level TEXT NOT NULL CHECK (level IN ('debug', 'info', 'warn', 'error')), + tag TEXT NOT NULL, + message TEXT NOT NULL, + detail TEXT, + created_at TEXT NOT NULL, + received_at TEXT NOT NULL +); + +CREATE INDEX idx_client_logs_user_id ON client_logs (user_name, id); +CREATE INDEX idx_client_logs_received ON client_logs (received_at); diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index ab5b9f5..96e3131 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -1,7 +1,7 @@ - %d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5level [%thread] %logger{36} - %msg%n + %d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5level [%thread] %logger{36} %X{requestId:-} - %msg%n diff --git a/src/test/kotlin/win/taptap/musicmanager/api/ApiRoutesTest.kt b/src/test/kotlin/win/taptap/musicmanager/api/ApiRoutesTest.kt index fd553ca..bd78b87 100644 --- a/src/test/kotlin/win/taptap/musicmanager/api/ApiRoutesTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/api/ApiRoutesTest.kt @@ -94,6 +94,7 @@ class ApiRoutesTest { assertEquals(0, info["deezerQuality"]!!.jsonPrimitive.int) assertTrue(info["loudnessEnabled"]!!.jsonPrimitive.content.toBoolean()) assertEquals("-16.0", info["loudnessTargetLufs"]!!.jsonPrimitive.content) + assertEquals(14, info["clientLogRetentionDays"]!!.jsonPrimitive.int) val malformed = client.post("/api/v1/moods") { bearer(token) @@ -105,6 +106,88 @@ class ApiRoutesTest { } } + @Test + fun `client diagnostics are private and redacted`() { + seededContext().use { seed -> + testApplication { + setupApp(seed) + val alice = client.token("alice", "alice-pw") + val bob = client.token("bob", "bob-pw") + val batch = """ + {"deviceId":"pixel-7","platform":"android","appVersion":"0.4.3", + "entries":[ + {"timestamp":1700000000000,"level":"warn","tag":"playback","message":"stalled","detail":"Authorization: Bearer keep-secret","sessionId":"run-1"}, + {"timestamp":1700000000000,"level":"error","tag":"api","message":"GET /stream -> 500","detail":"https://music/stream?token=abc123"} + ]} + """.trimIndent() + + val upload = client.post("/api/v1/logs") { + bearer(alice) + contentType(ContentType.Application.Json) + setBody(batch) + } + assertEquals(HttpStatusCode.OK, upload.status) + assertEquals(2, parse(upload.bodyAsText())["accepted"]!!.jsonPrimitive.int) + + val list = parse(client.get("/api/v1/logs") { bearer(alice) }.bodyAsText()) + val logs = list["logs"]!!.jsonArray + assertEquals(2, logs.size) + // Newest first: the error entry was inserted last. + assertEquals("api", logs[0].jsonObject.string("tag")) + assertEquals("error", logs[0].jsonObject.string("level")) + assertFalse(logs[0].jsonObject.string("detail").contains("abc123")) + assertFalse(logs[1].jsonObject.string("detail").contains("keep-secret")) + assertTrue(logs[1].jsonObject.string("detail").contains("***")) + + val levelFiltered = parse( + client.get("/api/v1/logs?level=warn") { bearer(alice) }.bodyAsText(), + ) + assertEquals(1, levelFiltered["logs"]!!.jsonArray.size) + + val bobLogs = parse(client.get("/api/v1/logs") { bearer(bob) }.bodyAsText()) + assertTrue(bobLogs["logs"]!!.jsonArray.isEmpty()) + + val forbidden = client.get("/api/v1/logs?user=alice") { bearer(bob) } + assertEquals(HttpStatusCode.Forbidden, forbidden.status) + + val adminView = parse( + client.get("/api/v1/logs?user=alice") { bearer(alice) }.bodyAsText(), + ) + assertEquals(2, adminView["logs"]!!.jsonArray.size) + + val invalid = client.post("/api/v1/logs") { + bearer(alice) + contentType(ContentType.Application.Json) + setBody( + """{"deviceId":"d","platform":"android","appVersion":"1", + "entries":[{"level":"verbose","tag":"x","message":"y"}]}""", + ) + } + assertEquals(HttpStatusCode.BadRequest, invalid.status) + + val many = (1..201).joinToString(",") { + """{"level":"info","tag":"x","message":"m"}""" + } + val tooMany = client.post("/api/v1/logs") { + bearer(alice) + contentType(ContentType.Application.Json) + setBody( + """{"deviceId":"d","platform":"android","appVersion":"1","entries":[$many]}""", + ) + } + assertEquals(HttpStatusCode.BadRequest, tooMany.status) + + val cleared = client.delete("/api/v1/logs") { bearer(alice) } + assertEquals(HttpStatusCode.OK, cleared.status) + assertEquals(2, parse(cleared.bodyAsText())["deleted"]!!.jsonPrimitive.int) + assertTrue( + parse(client.get("/api/v1/logs") { bearer(alice) }.bodyAsText())["logs"]!! + .jsonArray.isEmpty(), + ) + } + } + } + @Test fun `moods own their songs and are private`() { seededContext().use { seed -> @@ -644,6 +727,7 @@ class ApiRoutesTest { deezerQuality = 0, loudnessEnabled = true, loudnessTargetLufs = -16.0, + clientLogRetentionDays = 14, startScan = { false }, scanStatus = { ScanStatus(false, 0) }, ) diff --git a/src/test/kotlin/win/taptap/musicmanager/db/ClientLogRepositoryTest.kt b/src/test/kotlin/win/taptap/musicmanager/db/ClientLogRepositoryTest.kt new file mode 100644 index 0000000..7ff5f66 --- /dev/null +++ b/src/test/kotlin/win/taptap/musicmanager/db/ClientLogRepositoryTest.kt @@ -0,0 +1,110 @@ +package win.taptap.musicmanager.db + +import java.nio.file.Files +import java.time.Instant +import java.time.temporal.ChronoUnit +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq +import org.jetbrains.exposed.sql.transactions.transaction +import org.jetbrains.exposed.sql.update +import win.taptap.musicmanager.api.ClientLogBatchDto +import win.taptap.musicmanager.api.ClientLogEntryDto +import win.taptap.musicmanager.config.AppConfig +import win.taptap.musicmanager.config.UserAccount +import win.taptap.musicmanager.config.UsersConfig + +class ClientLogRepositoryTest { + + @Test + fun `credentials are redacted before they are stored`() { + withDatabase { db -> + assertEquals(1, ClientLogRepository.insert("alice", batch("playback", "Authorization: Bearer supersecret"))) + val row = ClientLogRepository.list("alice").single() + + assertFalse(row.message.contains("supersecret")) + assertFalse(row.detail.orEmpty().contains("supersecret")) + assertTrue(row.detail.orEmpty().contains("***")) + } + } + + @Test + fun `retention removes old rows and the per-user cap keeps the newest`() { + withDatabase { db -> + ClientLogRepository.insert("alice", batch("one", "first")) + ClientLogRepository.insert("alice", batch("two", "second")) + ClientLogRepository.insert("alice", batch("three", "third")) + + // Three rows, keep two: the newest two survive. + assertEquals(1, ClientLogRepository.prune(365, 2)) + assertEquals(listOf("third", "second"), ClientLogRepository.list("alice").map { it.message }) + + // Age the remaining rows past a one-day retention window. + transaction { + ClientLogsTable.update({ ClientLogsTable.userName eq "alice" }) { + it[receivedAt] = Instant.now().minus(10, ChronoUnit.DAYS).toString() + } + } + assertEquals(2, ClientLogRepository.prune(7, 2)) + assertTrue(ClientLogRepository.list("alice").isEmpty()) + } + } + + @Test + fun `users cannot see each other's diagnostics`() { + withDatabase { db -> + ClientLogRepository.insert("alice", batch("a", "alice line")) + ClientLogRepository.insert("bob", batch("b", "bob line")) + + assertEquals(listOf("alice line"), ClientLogRepository.list("alice").map { it.message }) + assertEquals(listOf("bob line"), ClientLogRepository.list("bob").map { it.message }) + + assertEquals(1, ClientLogRepository.delete("alice")) + assertTrue(ClientLogRepository.list("alice").isEmpty()) + assertEquals(1, ClientLogRepository.list("bob").size) + } + } + + private fun withDatabase(block: (DatabaseHandle) -> Unit) { + val dir = Files.createTempDirectory("client-logs-test") + val users = UsersConfig( + listOf( + UserAccount("alice", "alice-pw", admin = true), + UserAccount("bob", "bob-pw", admin = false), + ), + ) + val db = DatabaseFactory.connect( + AppConfig( + port = 0, + dataDir = dir, + usersFile = dir.resolve("users.yaml"), + ffmpegPath = "ffmpeg", + ), + ) + try { + ConfigRepository.sync(users) + block(db) + } finally { + db.close() + } + } + + private fun batch(tag: String, message: String): ClientLogBatchDto = + ClientLogBatchDto( + deviceId = "test-device", + platform = "desktop", + appVersion = "0.4.3", + entries = listOf( + ClientLogEntryDto( + timestamp = System.currentTimeMillis(), + level = "warn", + tag = tag, + message = message, + detail = "request failed: Authorization: Bearer supersecret", + sessionId = "run-test", + ), + ), + ) +}