Robustness: crash-safe migrations, stale-run sweep, optional prep tools; branded 404

- init_db applies each schema file with executescript (semicolons in triggers
  and string literals no longer break it) and records schema_version itself
  after each file, so a forgotten in-file bump can't re-run an ALTER and crash
  startup. Re-running is a no-op.
- On startup, sweep any job_runs left in 'running' (killed by a restart) to
  'interrupted' so the UI/history don't show a run stuck running forever.
- prep-audio.sh skips loudgain and cue_file cleanly when they aren't installed
  (they aren't in the base image) instead of failing on a missing binary and
  logging errors; pipeline-status.sh routes syslog through a guard so a missing
  `logger` doesn't spew into the job log.
- Add a branded 404 page (rendered to static HTML, no external runtime) via a
  custom exception handler that leaves all other responses, including the login
  redirect, untouched.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
andrew
2026-07-09 17:32:26 -06:00
parent 8f1fc458f6
commit fa86e992ad
5 changed files with 188 additions and 34 deletions
+45 -12
View File
@@ -1,4 +1,5 @@
import sqlite3
import time
from pathlib import Path
from sqlalchemy import create_engine, event, text
@@ -25,32 +26,64 @@ def _set_sqlite_pragmas(dbapi_connection, _record):
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
def _current_schema_version(conn) -> int:
def _raw_schema_version(conn: sqlite3.Connection) -> int:
try:
row = conn.execute(text("SELECT version FROM schema_version")).fetchone()
row = conn.execute("SELECT version FROM schema_version").fetchone()
return row[0] if row else 0
except Exception:
return 0
except sqlite3.OperationalError:
return 0 # schema_version table doesn't exist yet (fresh database)
def _set_schema_version(conn: sqlite3.Connection, version: int) -> None:
"""schema_version holds a single row (created by migration 0001). Set it
to the migration we just applied, so the version bump never depends on the
.sql file remembering to do it itself."""
cur = conn.execute("UPDATE schema_version SET version = ?", (version,))
if cur.rowcount == 0:
conn.execute("INSERT INTO schema_version (version) VALUES (?)", (version,))
def init_db() -> None:
"""Apply schema/*.sql migrations in order, tracked by schema_version.
Plain SQL files, not the `alembic` Python migration tool — see
docs/ARCHITECTURE.md for why."""
docs/ARCHITECTURE.md for why.
Each file is applied with executescript (so triggers and semicolons inside
string literals survive, unlike a naive split on ';'), and this function
sets schema_version after each file rather than trusting the file to. That
makes re-running safe: a file whose version is already recorded is skipped,
so a forgotten in-file version bump can't cause an ALTER to run twice and
crash startup."""
settings.app_db_path.parent.mkdir(parents=True, exist_ok=True)
migrations = sorted(SCHEMA_DIR.glob("*.sql"))
with engine.begin() as conn:
current = _current_schema_version(conn)
conn = sqlite3.connect(settings.app_db_path)
try:
current = _raw_schema_version(conn)
for path in migrations:
version = int(path.stem.split("_", 1)[0])
if version <= current:
continue
sql = path.read_text()
for statement in sql.split(";"):
statement = statement.strip()
if statement:
conn.execute(text(statement))
conn.executescript(path.read_text())
_set_schema_version(conn, version)
conn.commit()
finally:
conn.close()
def mark_interrupted_runs() -> None:
"""A job_run still marked 'running' at startup was killed mid-flight: the
subprocess does not survive a container restart. Mark those so the UI and
history don't show a run stuck 'running' forever. Runs once at startup,
before the scheduler can create any new runs."""
with engine.begin() as conn:
conn.execute(
text(
"UPDATE job_runs SET status = 'interrupted', "
"finished_at = COALESCE(finished_at, :now) WHERE status = 'running'"
),
{"now": time.time()},
)
def enable_beets_db_wal() -> None:
+18 -2
View File
@@ -1,17 +1,30 @@
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi import FastAPI, Request
from fastapi.exception_handlers import http_exception_handler
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.middleware.sessions import SessionMiddleware
from app.db import enable_beets_db_wal, init_db
from app.db import enable_beets_db_wal, init_db, mark_interrupted_runs
from app.routers import artist_casing, auth, credentials, dashboard, dedup, genres, health, import_, jobs, library, playlists
from app.security.session import resolve_session_secret
from app.services import scheduler_service
from app.settings import settings
log = logging.getLogger("alembic")
templates = Jinja2Templates(directory="app/templates")
async def not_found_or_default(request: Request, exc: StarletteHTTPException):
"""Render the branded 404 page for missing pages; leave every other
HTTP exception (including the 303 login redirect raised by require_auth)
to FastAPI's default handler so their status and headers are preserved."""
if exc.status_code == 404:
return templates.TemplateResponse(request, "404.html", {}, status_code=404)
return await http_exception_handler(request, exc)
def _warn_if_key_colocated_with_config() -> None:
@@ -39,6 +52,7 @@ def _warn_if_key_colocated_with_config() -> None:
async def lifespan(_app: FastAPI):
_warn_if_key_colocated_with_config()
init_db()
mark_interrupted_runs()
enable_beets_db_wal()
scheduler = scheduler_service.create_scheduler()
@@ -53,6 +67,8 @@ async def lifespan(_app: FastAPI):
def create_app() -> FastAPI:
app = FastAPI(title="alembic", lifespan=lifespan)
app.add_exception_handler(StarletteHTTPException, not_found_or_default)
app.add_middleware(SessionMiddleware, secret_key=resolve_session_secret())
app.mount("/static", StaticFiles(directory="app/static"), name="static")
+85
View File
@@ -0,0 +1,85 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Not found — alembic</title>
<link rel="icon" type="image/svg+xml" href="/static/icon.svg">
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="" />
<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@500;600;700&family=Manrope:wght@400;500;600;700;800&family=IBM+Plex+Mono:wght@500&display=swap" rel="stylesheet" />
<style>
@keyframes drift { from { transform: translate(0, 0) scale(1); } to { transform: translate(3vw, 4vw) scale(1.08); } }
@keyframes shimmer { from { background-position: 0% 50%; } to { background-position: 100% 50%; } }
@keyframes twinkle { 0%, 100% { opacity: 0.15; transform: scale(0.8) rotate(0deg); } 50% { opacity: 0.7; transform: scale(1.05) rotate(8deg); } }
body { margin: 0; }
.page-404::before {
content:''; position:fixed; inset:0; z-index:999; pointer-events:none; opacity:0.035; mix-blend-mode:overlay;
background-image:url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='120'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.9' numOctaves='2' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)'/%3E%3C/svg%3E");
}
</style>
</head>
<body>
<div class="page-404" style="margin:0; min-height:100vh; font-family:'Manrope','Segoe UI',system-ui,sans-serif; background:#08090d; color:#eef0f7; line-height:1.55; position:relative; overflow-x:hidden;">
<div style="position:fixed; width:42vw; height:42vw; top:-14vw; left:-10vw; border-radius:50%; filter:blur(110px); opacity:0.28; pointer-events:none; z-index:0; mix-blend-mode:screen; background:radial-gradient(circle, #a78bfa, transparent 70%); animation:drift 26s ease-in-out infinite alternate;"></div>
<div style="position:fixed; width:38vw; height:38vw; bottom:-16vw; right:-8vw; border-radius:50%; filter:blur(110px); opacity:0.28; pointer-events:none; z-index:0; mix-blend-mode:screen; background:radial-gradient(circle, #67e8f9, transparent 70%); animation:drift 26s ease-in-out infinite alternate; animation-delay:-8s;"></div>
<div style="position:fixed; width:30vw; height:30vw; top:30vh; right:20vw; border-radius:50%; filter:blur(110px); opacity:0.16; pointer-events:none; z-index:0; mix-blend-mode:screen; background:radial-gradient(circle, #f9a8d4, transparent 70%); animation:drift 26s ease-in-out infinite alternate; animation-delay:-16s;"></div>
<header style="position:sticky; top:0; z-index:50; background:rgba(8,9,13,0.72); backdrop-filter:blur(18px) saturate(140%); border-bottom:1px solid rgba(255,255,255,0.09);">
<div style="max-width:1180px; margin:0 auto; padding:0.85rem 1.5rem; display:flex; align-items:center; gap:1.75rem;">
<a href="/" style="display:inline-flex; align-items:center; gap:0.55rem; font-family:'Space Grotesk','Segoe UI',system-ui,sans-serif; font-weight:700; font-size:1.2rem; color:#eef0f7; text-decoration:none; flex-shrink:0;">
<svg width="22" height="22" viewBox="0 0 24 24" style="flex-shrink:0; filter:drop-shadow(0 0 10px rgba(167,139,250,0.45));">
<defs>
<linearGradient id="brandGrad" x1="0" y1="0" x2="24" y2="24">
<stop offset="0%" stop-color="#7dd3fc"></stop>
<stop offset="50%" stop-color="#a78bfa"></stop>
<stop offset="100%" stop-color="#f9a8d4"></stop>
</linearGradient>
</defs>
<path fill="url(#brandGrad)" d="M9 2.5h6v5.2l4.3 8.6a2.6 2.6 0 0 1-2.33 3.76H7.03A2.6 2.6 0 0 1 4.7 16.3L9 7.7V2.5z"></path>
<path d="M9 2.5h6M8.2 8.6h7.6" stroke="#08090d" stroke-width="0.9" stroke-linecap="round" opacity="0.35"></path>
</svg>
<span style="background:linear-gradient(115deg, #7dd3fc 0%, #a78bfa 32%, #f9a8d4 58%, #67e8f9 82%, #7dd3fc 100%); background-size:220% auto; -webkit-background-clip:text; background-clip:text; color:transparent; animation:shimmer 9s ease-in-out infinite alternate;">Alembic</span>
</a>
<nav style="display:flex; align-items:center; gap:0.15rem; flex:1; overflow-x:auto;">
<a href="/" style="color:#9799ac; font-size:0.88rem; font-weight:600; padding:0.45rem 0.85rem; border-radius:999px; white-space:nowrap; text-decoration:none;">Dashboard</a>
<a href="/playlists" style="color:#9799ac; font-size:0.88rem; font-weight:600; padding:0.45rem 0.85rem; border-radius:999px; white-space:nowrap; text-decoration:none;">Playlists</a>
<a href="/library" style="color:#9799ac; font-size:0.88rem; font-weight:600; padding:0.45rem 0.85rem; border-radius:999px; white-space:nowrap; text-decoration:none;">Library</a>
<a href="/dedup" style="color:#9799ac; font-size:0.88rem; font-weight:600; padding:0.45rem 0.85rem; border-radius:999px; white-space:nowrap; text-decoration:none;">Dedup</a>
<a href="/genres" style="color:#9799ac; font-size:0.88rem; font-weight:600; padding:0.45rem 0.85rem; border-radius:999px; white-space:nowrap; text-decoration:none;">Genres</a>
<a href="/import" style="color:#9799ac; font-size:0.88rem; font-weight:600; padding:0.45rem 0.85rem; border-radius:999px; white-space:nowrap; text-decoration:none;">Import</a>
<a href="/settings/credentials" style="color:#9799ac; font-size:0.88rem; font-weight:600; padding:0.45rem 0.85rem; border-radius:999px; white-space:nowrap; text-decoration:none;">Settings</a>
</nav>
</div>
</header>
<main style="max-width:1180px; margin:0 auto; padding:2.5rem 1.5rem 5rem; position:relative; z-index:1; min-height:calc(100vh - 68px); display:flex; align-items:center; justify-content:center;">
<div style="text-align:center; max-width:640px; position:relative; padding:1rem 1.5rem;">
<svg width="16" height="16" viewBox="0 0 24 24" style="position:absolute; top:-6px; left:8%; opacity:0.6; animation:twinkle 3.2s ease-in-out infinite;">
<path fill="#c9c4fa" d="M12 0c.6 5.7 1.8 8.6 4.2 10.6 2.4 2 5.3 2.6 7.8 3.4-2.5.8-5.4 1.4-7.8 3.4-2.4 2-3.6 4.9-4.2 10.6-.6-5.7-1.8-8.6-4.2-10.6C5.4 15.4 2.5 14.8 0 14c2.5-.8 5.4-1.4 7.8-3.4C10.2 8.6 11.4 5.7 12 0Z"></path>
</svg>
<svg width="20" height="20" viewBox="0 0 24 24" style="position:absolute; top:40px; right:6%; opacity:0.45; animation:twinkle 3.2s ease-in-out infinite; animation-delay:-1.4s;">
<path fill="#c9c4fa" d="M12 0c.6 5.7 1.8 8.6 4.2 10.6 2.4 2 5.3 2.6 7.8 3.4-2.5.8-5.4 1.4-7.8 3.4-2.4 2-3.6 4.9-4.2 10.6-.6-5.7-1.8-8.6-4.2-10.6C5.4 15.4 2.5 14.8 0 14c2.5-.8 5.4-1.4 7.8-3.4C10.2 8.6 11.4 5.7 12 0Z"></path>
</svg>
<svg width="12" height="12" viewBox="0 0 24 24" style="position:absolute; bottom:10px; left:20%; opacity:0.4; animation:twinkle 3.2s ease-in-out infinite; animation-delay:-2.2s;">
<path fill="#c9c4fa" d="M12 0c.6 5.7 1.8 8.6 4.2 10.6 2.4 2 5.3 2.6 7.8 3.4-2.5.8-5.4 1.4-7.8 3.4-2.4 2-3.6 4.9-4.2 10.6-.6-5.7-1.8-8.6-4.2-10.6C5.4 15.4 2.5 14.8 0 14c2.5-.8 5.4-1.4 7.8-3.4C10.2 8.6 11.4 5.7 12 0Z"></path>
</svg>
<div style="font-family:'IBM Plex Mono',ui-monospace,monospace; font-size:0.78rem; text-transform:uppercase; letter-spacing:0.14em; color:#6d6f83; margin-bottom:0.75rem;">Error 404 &middot; No signal</div>
<div style="font-family:'Space Grotesk','Segoe UI',system-ui,sans-serif; font-weight:700; font-size:clamp(5rem, 15vw, 8.5rem); line-height:1; letter-spacing:-0.02em; background:linear-gradient(115deg, #7dd3fc 0%, #a78bfa 32%, #f9a8d4 58%, #67e8f9 82%, #7dd3fc 100%); background-size:220% auto; -webkit-background-clip:text; background-clip:text; color:transparent; animation:shimmer 9s ease-in-out infinite alternate; margin-bottom:0.25rem;">404</div>
<h1 style="font-family:'Space Grotesk','Segoe UI',system-ui,sans-serif; font-weight:600; letter-spacing:-0.01em; margin:0 0 0.6rem; color:#eef0f7; font-size:clamp(1.4rem, 2.4vw, 1.8rem);">This track didn't survive distillation.</h1>
<p style="margin:0 0 2rem; color:#9799ac; font-size:0.98rem;">Nothing's left in the flask at this address — it may have evaporated, moved, or never existed.</p>
<div style="display:flex; gap:0.6rem; flex-wrap:wrap; align-items:center; justify-content:center;">
<a href="/" style="display:inline-flex; align-items:center; justify-content:center; gap:0.4rem; font-family:inherit; font-weight:700; font-size:0.9rem; padding:0.65rem 1.3rem; border-radius:999px; border:none; color:#08090d; background:linear-gradient(115deg, #7dd3fc 0%, #a78bfa 32%, #f9a8d4 58%, #67e8f9 82%, #7dd3fc 100%); background-size:220% auto; box-shadow:0 10px 30px -12px rgba(167,139,250,0.6); text-decoration:none; cursor:pointer;">Back to dashboard</a>
<a href="/playlists" style="display:inline-flex; align-items:center; justify-content:center; gap:0.4rem; font-family:inherit; font-weight:700; font-size:0.9rem; padding:0.65rem 1.3rem; border-radius:999px; border:1px solid rgba(255,255,255,0.16); background:rgba(255,255,255,0.035); color:#eef0f7; text-decoration:none; cursor:pointer;">View playlists</a>
</div>
</div>
</main>
</div>
</body>
</html>
+11 -3
View File
@@ -43,6 +43,14 @@ mark_info() { printf " %s\n" "$*"; }
section() { printf "\n▎ %s\n" "$*"; }
# syslog / `logger` is not present in the container image. Only call it if it
# exists, so these lines don't spew "logger: command not found" into the job
# log on every run. Always returns success.
slog() {
command -v logger >/dev/null 2>&1 || return 0
logger -t sldl-pipeline "$@"
}
# Format an age in seconds as "Nh", "Nd", "Nw", "Nmo".
human_age() {
local s=$1
@@ -375,16 +383,16 @@ ${BANNER_TALLY}" "$OUT"
# Always mirror the one-line summary to syslog so journalctl shows last-run
# state even if Telegram is broken. Warnings get a separate WARN tag.
logger -t sldl-pipeline "status: $OK ok, $WARN warn, $SKIP skip (see $OUT)"
slog "status: $OK ok, $WARN warn, $SKIP skip (see $OUT)"
if (( WARN > 0 )); then
logger -t sldl-pipeline "WARN: pipeline-status flagged $WARN issue(s) — see $OUT"
slog "WARN: pipeline-status flagged $WARN issue(s) — see $OUT"
fi
# Send to Telegram. If it fails, log the failure to syslog so the absence of
# a message in the chat has a corresponding journal entry to grep for.
if [[ -x ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh ]]; then
if ! ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh < "$OUT" 2>/tmp/tg-err; then
logger -t sldl-pipeline "WARN: Telegram send failed: $(cat /tmp/tg-err 2>/dev/null | head -c 200)"
slog "WARN: Telegram send failed: $(cat /tmp/tg-err 2>/dev/null | head -c 200)"
rm -f /tmp/tg-err
fi
fi
+29 -17
View File
@@ -26,25 +26,37 @@ prep_audio() {
return 0
fi
echo "[prep-audio] loudgain (track-mode RG, clip-safe) on ${#files[@]} file(s)" >>"$log"
# -q quiet, -k lower track gain to avoid clipping >-1 dBFS,
# -s e write extended ReplayGain2 + R128 tags, -I 4 ID3v2.4 for MP3.
# No -a, so each file gets its own track gain (track-mode).
if ! loudgain -q -k -s e -I 4 "${files[@]}" >>"$log" 2>&1; then
echo "[prep-audio] loudgain returned non-zero (see log)" >>"$log"
# loudgain (ReplayGain) and cue_file (Liquidsoap autocue) are optional. They
# are not in the base image, so only run them if present -- otherwise skip
# cleanly with a note instead of failing on a missing binary. Install them in
# the image if you want ReplayGain/autocue tags written at prep time.
if command -v loudgain >/dev/null 2>&1; then
echo "[prep-audio] loudgain (track-mode RG, clip-safe) on ${#files[@]} file(s)" >>"$log"
# -q quiet, -k lower track gain to avoid clipping >-1 dBFS,
# -s e write extended ReplayGain2 + R128 tags, -I 4 ID3v2.4 for MP3.
# No -a, so each file gets its own track gain (track-mode).
if ! loudgain -q -k -s e -I 4 "${files[@]}" >>"$log" 2>&1; then
echo "[prep-audio] loudgain returned non-zero (see log)" >>"$log"
fi
else
echo "[prep-audio] loudgain not installed, skipping ReplayGain" >>"$log"
fi
echo "[prep-audio] cue_file (autocue liq_* tags) on ${#files[@]} file(s)" >>"$log"
# -w write tags into file, -k true-peak clipping prevention.
# Skips files that already have liq_cue_file=true (use -f to force).
local failed=0
for f in "${files[@]}"; do
if ! cue_file -w -k "$f" >>"$log" 2>&1; then
echo "[prep-audio] cue_file failed on $f" >>"$log"
failed=$((failed + 1))
fi
done
[[ $failed -gt 0 ]] && echo "[prep-audio] cue_file errors: $failed" >>"$log"
if command -v cue_file >/dev/null 2>&1; then
echo "[prep-audio] cue_file (autocue liq_* tags) on ${#files[@]} file(s)" >>"$log"
# -w write tags into file, -k true-peak clipping prevention.
# Skips files that already have liq_cue_file=true (use -f to force).
local failed=0
for f in "${files[@]}"; do
if ! cue_file -w -k "$f" >>"$log" 2>&1; then
echo "[prep-audio] cue_file failed on $f" >>"$log"
failed=$((failed + 1))
fi
done
[[ $failed -gt 0 ]] && echo "[prep-audio] cue_file errors: $failed" >>"$log"
else
echo "[prep-audio] cue_file not installed, skipping autocue tags" >>"$log"
fi
echo "${#files[@]}"
}