#!/bin/bash # pipeline-status.sh — daily health check for the music pipeline. # # Writes a one-screen summary to ${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log (overwritten daily) # and ships it to Telegram via notify-telegram.sh. # # Formatting note: the digest uses Telegram HTML entities (, , , # ) — the brand's chrome/dot-badge language translated into what # Telegram can actually render (no color, no custom font, so status "dots" # become colored circle emoji). Deliberately NO /monospace column layout: # a code block forces fixed-width columns that wrap into unreadable garbage on # a phone ("54 tracks in / M3U"). Instead every row is plain reflowing text — # a bold label + " — value" — so it wraps cleanly at any screen width. This # REQUIRES notify-telegram.sh to be called with --html (parse_mode=HTML) — # sent as plain text the tags would show up literally. All free-text going # through mark_ok/mark_warn/mark_skip/section is escaped (esc()) for &, <, > # so a stray angle bracket in a log line can't break the HTML parse and # swallow the whole message. # # Reports on: # - Sibling service reachability (navidrome) via HTTP, not docker ps — # alembic has no Docker socket access. slskd is deliberately NOT checked: # it is not part of the pipeline (sldl is its own Soulseek client) — it # runs on the host purely as the user's own file-sharing presence. # - Today's runs: playlist syncs, Bandcamp sync, manual imports, dedup # - Weekly maintenance freshness: strip-mb-tags, strip-watermark-art, # scrub-watermark-text, clean-sldl-index, spotify-genre # - Daily dedup freshness # - Monthly mp3→flac upgrade freshness # - Auth & state: Bandcamp cookie expiry, Qobuz token liveness, VPN egress # - Storage, Navidrome scan freshness, library size by format # # Only flags SCRIPT-level failures, not per-file Soulseek errors (those are # normal — peers go offline, connections reset). # set -u only, NOT -e/pipefail, on purpose. This script is read-only: it # assembles a status digest from ~30 independent probes (grep over logs, beet # queries, curl health checks, optional-credential file reads). Each probe is # allowed to fail individually and report a warn/skip line; that's the whole # design. Under set -e a single missing optional file (e.g. qobuz/app_id) or a # no-match grep would abort the entire run and send NO digest at all, which is # the opposite of what a health report should do. It writes no library state, # so there is nothing to leave half-applied. set -u # /opt/venv/bin first: `beet` lives in the app venv. Without it every beet # probe here ("added today", "Library by format", the mp3-now count) silently # came up empty behind its 2>/dev/null. PATH=/opt/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin OUT=${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log TODAY=$(date +%Y%m%d) NOW_TS=$(date +%s) NAVIDROME_ENV="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/navidrome/admin.env" [ -f "$NAVIDROME_ENV" ] && source "$NAVIDROME_ENV" ND_BASE="${ND_BASE:-http://navidrome:4533}" ND_USER="${ND_USER:-}" ND_PASS="${ND_PASS:-}" # Tally counters for the header banner. Body collected first, then prepended. OK=0 WARN=0 SKIP=0 # Escapes &, <, > so free text (log snippets, exception messages, filenames) # can't be mistaken for an HTML entity by Telegram's parser. esc() { local s=$1 s=${s//&/&} s=${s///>} printf '%s' "$s" } # One status row: a colored dot, a bold label, and (optionally) " — value". # Plain proportional text — NO padding, NO monospace — so it reflows on mobile # instead of wrapping mid-column. The dot is the brand's badge-state color, # emoji being Telegram's only color channel. _row() { local dot=$1 label=$2 detail=${3:-} if [[ -n "$detail" ]]; then printf "%s %s — %s\n" "$dot" "$(esc "$label")" "$(esc "$detail")" else printf "%s %s\n" "$dot" "$(esc "$label")" fi } mark_ok() { OK=$((OK+1)); _row "🟢" "$@"; } mark_warn() { WARN=$((WARN+1)); _row "🟠" "$@"; } mark_skip() { SKIP=$((SKIP+1)); _row "⚪" "$@"; } # Free-text info line (no dot, no counter) — for sub-breakdowns. mark_info() { printf " %s\n" "$(esc "$*")"; } # Bold section header (reads like the web app's ). No — the rows # under it are plain reflowing text. section() { printf "\n▎ %s\n" "$(esc "$*")" } # syslog / `logger` is not present in the container image. Only call it if it # exists, so these lines don't spew "logger: command not found" into the job # log on every run. Always returns success. slog() { command -v logger >/dev/null 2>&1 || return 0 logger -t sldl-pipeline "$@" } # Format an age in seconds as "Nh", "Nd", "Nw", "Nmo". human_age() { local s=$1 if (( s < 3600 )); then echo "$((s/60))m ago" elif (( s < 86400 )); then echo "$((s/3600))h ago" elif (( s < 86400*14 )); then echo "$((s/86400))d ago" elif (( s < 86400*60 )); then echo "$((s/86400/7))w ago" else echo "$((s/86400/30))mo ago" fi } # Per-playlist log status. playlist_status() { local log="$1" if grep -q "=== Finished playlist run" "$log" 2>/dev/null; then # awk picks just the count after "with"; sidesteps the "M3U" digit-3 trap # that bites grep -oE '[0-9]+'. local m3u m3u=$(awk 'match($0, /updated with [0-9]+ tracks/) { n=substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", n); last=n } END { print last }' "$log") echo "OK|${m3u:-0} tracks" elif grep -q "=== Starting playlist run" "$log" 2>/dev/null; then echo "WARN|started but never finished" else echo "WARN|no Starting line" fi } bandcamp_status() { local log="$1" if grep -q "=== Bandcamp sync done" "$log" 2>/dev/null; then local s s=$(grep -oE "\[bandcamp\] === [0-9]+ new[^]]*" "$log" | tail -1 | sed 's/\[bandcamp\] === //; s/ ===$//') echo "OK|${s:-completed}" else echo "WARN|started but never finished" fi } manual_import_status() { local log="$1" local n # || true, not || echo 0: grep -c already prints "0" on no-match (and exits # 1), so || echo 0 produced a two-line "0\n0" that broke the -ge test. n=$(grep -c "=== Finished manual import" "$log" 2>/dev/null || true) if [[ "${n:-0}" -ge 1 ]]; then echo "OK|$n run(s)" else echo "WARN|started but never finished" fi } dedup_today_status() { local log="$1" if grep -q "=== Summary:" "$log" 2>/dev/null; then # Format: "=== Summary: N dup groups (parens with commas), N kept, N to delete ===" # Strip the wrapper and the parenthesized breakdown so the line stays short. local s s=$(grep "=== Summary:" "$log" | tail -1 \ | sed -E 's/.*=== Summary: //; s/ ===.*//; s/ $[^)]*$//') echo "OK|${s:-completed (no summary line)}" else echo "WARN|started but never finished" fi } # ---- Build report ---- { echo "__HEADER_PLACEHOLDER__" # 1. Sibling service health # alembic deliberately has no Docker socket access (nothing left needs it # once sldl/beet run in-process), so these are HTTP reachability checks # against each service's own endpoint rather than `docker ps` container # state — arguably a better signal anyway (checks the service actually # answers, not just that the container process exists). section "Services" check_http() { local name="$1" url="$2" if curl -s -o /dev/null --connect-timeout 3 --max-time 6 "$url"; then mark_ok "$name" "reachable" else mark_warn "$name" "unreachable at $url" fi } check_http navidrome "http://navidrome:4533/rest/ping.view" # 2. Today's runs # Allowlist-based: only Spotify playlists, Bandcamp, manual import, and # dedup count as "runs." Everything else (laptop-export, normalize-casing, # gen-djmix, gen-vgm, fix-empty-album, fingerprint-index, backfill, replace, # convert-m4a, fred-mixes, dj-import-*, weekly tag-hygiene) is either # boring/silent maintenance or already covered in the Maintenance section. section "Today's runs" any_today=0 newest_dedup_today=$(ls -1t ${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$TODAY-*.log 2>/dev/null | head -1) # Known Spotify playlist names — derive from configs/ directory so adding a # new playlist auto-includes it here without editing this script. PLAYLIST_NAMES=$(ls ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/*.conf 2>/dev/null \ | xargs -n1 basename 2>/dev/null \ | sed 's/\.conf$//' \ | grep -v '^_' || true) for log in ${ALEMBIC_CONFIG_DIR:-/config}/logs/*-$TODAY.log; do [[ -f "$log" ]] || continue name=$(basename "$log" .log) short=${name%-$TODAY} case "$short" in manual-import) any_today=1 IFS='|' read -r tag detail < <(manual_import_status "$log") ;; bandcamp) any_today=1 IFS='|' read -r tag detail < <(bandcamp_status "$log") ;; *) # Is this a Spotify playlist log? Check against PLAYLIST_NAMES. if echo "$PLAYLIST_NAMES" | grep -qxF "$short"; then any_today=1 IFS='|' read -r tag detail < <(playlist_status "$log") else continue # not a "run" — silent / belongs in Maintenance / irrelevant fi ;; esac if [[ "$tag" == "OK" ]]; then mark_ok "$short" "$detail"; else mark_warn "$short" "$detail"; fi done if [[ -n "$newest_dedup_today" ]]; then any_today=1 IFS='|' read -r tag detail < <(dedup_today_status "$newest_dedup_today") if [[ "$tag" == "OK" ]]; then mark_ok "dedup" "$detail"; else mark_warn "dedup" "$detail"; fi fi [[ "$any_today" -eq 0 ]] && mark_info "(nothing has run yet today)" # 2b. Nightly download tally — count tracks added to beets in the last 24h. # Captures the overnight playlist sweep + any daytime Bandcamp/manual imports. # Uses beets' `added` timestamp, so it's accurate regardless of how files # arrived (Spotify pipeline, Bandcamp, manual). Per-playlist breakdown # follows when ≥1 track was added with that grouping today. ADDED_TODAY=$(beet ls -f '$grouping' \ "added:$(date +%Y-%m-%d).." 2>/dev/null || true) added_total=$(echo -n "$ADDED_TODAY" | grep -c '^' || true) mark_ok "added today" "$added_total new tracks in beets" if [[ "$added_total" -gt 0 ]]; then # Per-playlist breakdown. Empty $grouping (bandcamp/manual) shown as "(none)". # Piped through the same &/ escaping as esc(), since this prints # straight into the message without going through _row. echo "$ADDED_TODAY" \ | awk '{ if ($0 == "") print "(none)"; else print }' \ | sort | uniq -c | sort -rn \ | awk '{ c=$1; g=$2; for(i=3;i<=NF;i++) g=g" "$i; printf " • %s (%d)\n", g, c }' \ | sed 's/&/\&/g; s//\>/g' fi # 3. Maintenance freshness # Read from the app's job_runs table (the scheduler's own record) instead of # globbing for log files. Jobs run through pipeline_runner write their logs # to logs//.log, so the old cron-era filename globs # matched nothing for jobs that don't also write their own log, and healthy # jobs were reported as "never run yet". The DB also distinguishes lock # skips and failures, which a missing log file can't. # Limits: weekly tasks should be <9 days old; daily <2; monthly <35. section "Maintenance (last run)" declare -A MAINT_LIMIT=( [strip-mb-tags]=$((9*86400)) [strip-watermark-art]=$((9*86400)) [scrub-watermark-text]=$((9*86400)) [clean-sldl-index]=$((9*86400)) [clear-bad-genres]=$((9*86400)) [spotify-genre]=$((9*86400)) [dedup-library]=$((2*86400)) [upgrade-mp3-to-flac]=$((35*86400)) ) # Emits one line per task: task|latest_status|latest_age_s|success_age_s|success_log # (ages are -1 when there is no such run). Read-only DB open; prints nothing # if the DB is missing so every task falls through to "never run yet". maint_rows=$(python3 - "${ALEMBIC_CONFIG_DIR:-/config}/alembic.db" <<'PYEOF' import sqlite3 import sys import time # Digest label -> job_runs.job_key. The scheduled genre refresh records under # genre:run (via genre_review_service) and dedup under dedup:scan (via # dedup_review_service); everything else is its MAINTENANCE_JOBS key. KEYS = [ ("strip-mb-tags", "maintenance:strip_mb_tags"), ("strip-watermark-art", "maintenance:strip_watermark_art"), ("scrub-watermark-text", "maintenance:scrub_watermark_text"), ("clean-sldl-index", "maintenance:clean_sldl_index"), ("clear-bad-genres", "maintenance:clear_bad_genres"), ("spotify-genre", "genre:run"), ("dedup-library", "dedup:scan"), ("upgrade-mp3-to-flac", "maintenance:upgrade_mp3_to_flac"), ] now = time.time() try: conn = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True) conn.execute("SELECT 1 FROM job_runs LIMIT 1") except sqlite3.Error: sys.exit(0) for task, key in KEYS: latest = conn.execute( "SELECT status, started_at FROM job_runs WHERE job_key = ? ORDER BY started_at DESC LIMIT 1", (key,), ).fetchone() if latest is None: print(f"{task}|none|-1|-1|") continue success = conn.execute( "SELECT started_at, log_path FROM job_runs WHERE job_key = ? AND status = 'success' " "ORDER BY started_at DESC LIMIT 1", (key,), ).fetchone() s_age = int(now - success[0]) if success else -1 s_log = (success[1] or "") if success else "" print(f"{task}|{latest[0]}|{int(now - latest[1])}|{s_age}|{s_log}") PYEOF ) for task in strip-mb-tags strip-watermark-art scrub-watermark-text clean-sldl-index clear-bad-genres spotify-genre dedup-library upgrade-mp3-to-flac; do row=$(grep "^${task}|" <<< "$maint_rows" || true) IFS='|' read -r _ latest_status latest_age age_s log <<< "$row" if [[ -z "$row" || "$latest_status" == "none" ]]; then mark_skip "$task" "never run yet" continue fi if [[ "$age_s" -lt 0 ]]; then # attempted, but never once succeeded -- say what the last attempt did case "$latest_status" in skipped_lock) note="skipped, lock busy" ;; running) note="running now" ;; *) note="$latest_status" ;; esac mark_warn "$task" "never succeeded (last attempt $(human_age "$latest_age"): $note)" continue fi age_h=$(human_age "$age_s") snip="" if [[ -n "$log" && -f "$log" ]]; then case "$task" in spotify-genre) snip=$(grep -oE "written: [0-9]+, unchanged: [0-9]+" "$log" | tail -1) ;; clear-bad-genres) snip=$(grep -oE "rewrote: [0-9]+ tracks, blanked: [0-9]+ tracks|(would-clear|cleared): [0-9]+ tracks" "$log" | tail -1) ;; clean-sldl-index) snip=$(grep -oE "[0-9]+ m3us, total [0-9]+ kept, [0-9]+ dropped" "$log" | tail -1) ;; strip-watermark-art) snip=$(grep -oE "stripped from [0-9]+ tracks|no images shared" "$log" | tail -1) ;; scrub-watermark-text) snip=$(grep -oE "stripped [0-9]+ frame$s$ across [0-9]+ file" "$log" | tail -1) ;; strip-mb-tags) snip=$(grep -oE "Done\. Log:" "$log" | head -1 | sed 's/Done\. Log:/done/') ;; dedup-library) snip=$(grep "=== Summary:" "$log" | tail -1 \ | sed -E 's/.*=== Summary: //; s/ ===.*//; s/ $[^)]*$//') ;; upgrade-mp3-to-flac) # The log's "to attempt" count is frozen at the last monthly run, so on # its own it looks stale against the live "Library by format" section. # Pair it with the current format:mp3 count so the drop (upgrades that # have landed since) is visible instead of looking like a mismatch. attempted=$(grep -oE "[0-9]+ MP3 tracks to attempt" "$log" | grep -oE "^[0-9]+" | tail -1) mp3_now=$(beet ls 'format:mp3' 2>/dev/null | grep -c '^') snip="${attempted:-?} attempted · ${mp3_now} MP3 now" ;; esac fi # Age/snippet describe the last SUCCESS; if a newer attempt failed or was # lock-skipped, say so rather than hiding it behind the healthy line. case "$latest_status" in failed) snip="${snip:+$snip, }latest attempt failed" ;; skipped_lock) snip="${snip:+$snip, }latest attempt skipped, lock busy" ;; esac detail="$age_h" [[ -n "$snip" ]] && detail="$age_h · $snip" if (( age_s > MAINT_LIMIT[$task] )) || [[ "$latest_status" == "failed" ]]; then mark_warn "$task" "$detail" else mark_ok "$task" "$detail" fi done # 4. Auth & state section "Auth & state" # Bandcamp cookie if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt ]]; then exp=$(awk -F'\t' '$6=="identity" {print $5; exit}' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt) if [[ -n "${exp:-}" && "$exp" =~ ^[0-9]+$ && "$exp" -gt 0 ]]; then days=$(( (exp - NOW_TS) / 86400 )) if (( days < 14 )); then mark_warn "Bandcamp cookie" "$days days left — re-export soon" else mark_ok "Bandcamp cookie" "$days days left" fi else mark_warn "Bandcamp cookie" "could not parse expiry" fi else mark_warn "Bandcamp cookie" "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)" fi # Qobuz Web Player token (buy-link enrich cascade #2). The token is opaque # (no decodable expiry), so probe it live: a cheap authed search returns 200 # while valid, 401 once it's logged out — at which point enrich-buy-url.py # silently drops Qobuz and falls through to iTunes. if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token ]]; then q_token=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token) # Guard the app_id read with a file-exists check: `< missing 2>/dev/null` # still leaks the shell's own "No such file" redirection error (the redirect # is opened before 2>/dev/null takes effect). Most setups have a token but # no separate app_id, so this fired on every run. q_appid=798273057 q_appid_file="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/app_id" [[ -f "$q_appid_file" ]] && q_appid=$(tr -d '\r\n' < "$q_appid_file") q_code=$(curl -s -o /dev/null --connect-timeout 5 --max-time 12 -A "Mozilla/5.0" \ -H "X-App-Id: $q_appid" -H "X-User-Auth-Token: $q_token" -w '%{http_code}' \ "https://www.qobuz.com/api.json/0.2/track/search?query=test&limit=1&app_id=$q_appid" 2>/dev/null) case "$q_code" in 200) mark_ok "Qobuz token" "valid (buy-link lookup live)" ;; 401) mark_warn "Qobuz token" "EXPIRED — re-export X-User-Auth-Token to ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token" ;; *) mark_warn "Qobuz token" "check failed (HTTP ${q_code:-none})" ;; esac else mark_warn "Qobuz token" "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)" fi # VPN egress # alembic itself rides gluetun's network namespace (network_mode: # service:gluetun), so its own outbound traffic IS gluetun's traffic — # checking alembic's own egress IP directly is a more direct signal than # asking Docker to introspect gluetun's health, and needs no Docker socket. egress_ip=$(curl -s --connect-timeout 5 --max-time 10 https://ifconfig.me 2>/dev/null) if [[ -n "$egress_ip" ]]; then mark_ok "VPN egress" "$egress_ip" else mark_warn "VPN egress" "could not determine egress IP — VPN may be down" fi # 5. Storage section "Storage" while read -r mp pct used total; do pct_n=${pct%\%} if (( pct_n > 90 )); then mark_warn "$mp" "$pct used ($used of $total)" else mark_ok "$mp" "$pct used ($used of $total)" fi done < <(df -h ${MUSIC_DATA_DIR:-/data/music} / 2>/dev/null | awk 'NR>1 && $1!~/tmpfs/ {print $6, $5, $3, $2}') # 6. Navidrome section "Navidrome" # Password via stdin (p@-), not argv, so it isn't exposed in ps/proc. ND_RESP=$(printf '%s' "$ND_PASS" | curl -s --connect-timeout 5 -G "$ND_BASE/rest/getScanStatus.view" \ --data-urlencode "u=$ND_USER" \ --data-urlencode "p@-" \ --data-urlencode 'v=1.16.0' --data-urlencode 'c=status' --data-urlencode 'f=json' 2>/dev/null) ND_LINE=$(echo "$ND_RESP" | python3 -c " import sys, json try: d = json.load(sys.stdin)['subsonic-response'] if d.get('status') == 'ok': s = d.get('scanStatus', {}) cnt = s.get('count', '?') cnt_str = f'{cnt:,}' if isinstance(cnt, int) else str(cnt) print(f\"OK|last scan {s.get('lastScan','?')[:19]} · {cnt_str} items\") else: print(f\"WARN|status={d.get('status')}\") except Exception as e: print(f\"WARN|unreachable: {e}\") " 2>/dev/null) state=${ND_LINE%%|*}; detail=${ND_LINE##*|} if [[ "$state" == "OK" ]]; then mark_ok "Navidrome" "$detail"; else mark_warn "Navidrome" "$detail"; fi # 7. Library by format (info-only, no status dot) section "Library by format" beet ls -f '$format' 2>/dev/null | sort | uniq -c | sort -rn \ | awk '{printf " • %s — %d tracks\n", $2, $1}' \ | sed 's/&/\&/g; s//\>/g' } > "$OUT" # ---- Build header banner and prepend ---- # Bold brand mark (⚗️ is literally the "ALEMBIC" unicode glyph — no generic # music-note stand-in needed), a hostname chip in , and an italic tally # that reuses the same 🟢/🟠/⚪ dots as the body. When something needs # attention, a callout mirrors the web app's .notice.warning # panel; an all-clear run gets the calm .notice.success equivalent instead. HOSTNAME_SHORT=$(hostname -s) DATE_HUMAN=$(TZ="${TZ:-UTC}" date '+%a %Y-%m-%d %H:%M %Z') HEADER="⚗️ Alembic · music pipeline · ${HOSTNAME_SHORT} · ${DATE_HUMAN} 🟢 ${OK} OK 🟠 ${WARN} warn ⚪ ${SKIP} skip" if (( WARN > 0 )); then HEADER="${HEADER} 🟠 ${WARN} issue(s) need attention — see below" else HEADER="${HEADER} 🟢 All clear — nothing needs attention." fi # Swap the __HEADER_PLACEHOLDER__ line for the (2-3 line) banner above. A # straight prepend, not sed 1c — the banner's line count varies with the # blockquote, which sed's `c` command can't take as a variable easily. tail -n +2 "$OUT" > "${OUT}.body" { printf '%s\n' "$HEADER"; cat "${OUT}.body"; } > "$OUT" rm -f "${OUT}.body" # Always mirror the one-line summary to syslog so journalctl shows last-run # state even if Telegram is broken. Warnings get a separate WARN tag. slog "status: $OK ok, $WARN warn, $SKIP skip (see $OUT)" if (( WARN > 0 )); then slog "WARN: pipeline-status flagged $WARN issue(s) — see $OUT" fi # Send to Telegram with --html: the digest is Telegram-HTML (see header # comment), so notify-telegram.sh must send it with parse_mode=HTML. If the # send fails, log to syslog so the absence of a message in the chat has a # corresponding journal entry to grep for. if [[ -x ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh ]]; then if ! ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh --html < "$OUT" 2>/tmp/tg-err; then slog "WARN: Telegram send failed: $(cat /tmp/tg-err 2>/dev/null | head -c 200)" rm -f /tmp/tg-err fi fi