andrew
7a49e2d507
The Telegram digest reported healthy weekly jobs as "never run yet": its maintenance section still globbed for cron-era per-script log filenames (clean-index-*.log etc.) that jobs run through pipeline_runner no longer write. It now reads job_runs, the scheduler's own record, so it reports the last success (age + summary snippet from that run's log), flags a newer failed or lock-skipped attempt on the same line, and distinguishes "never succeeded (last attempt: skipped, lock busy)" from genuinely never scheduled. Also matched the clear-bad-genres snippet grep to the script's current summary wording, and dropped the now-unused newest_log helper. The DB also showed WHY two jobs had never run: on Sunday the 08:30 strip-mb-tags run took 10m19s while holding the shared pipeline lock, so strip-watermark-art (08:35) and scrub-watermark-text (08:40) hit flock-style instant skip and lost their only slot of the week -- the 08:40 job missed by 19 seconds. Scheduled runs now wait up to 30 minutes for the lock (SCHEDULED_LOCK_WAIT_SECONDS) and only then record skipped_lock, so fixed-time blocks queue instead of starving; manual "Run now" keeps the instant skip since a person expects an immediate answer. Tests: scheduled-run queueing, lock-wait timeout, and manual instant-skip. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
461 lines
20 KiB
Bash
Executable File
461 lines
20 KiB
Bash
Executable File
#!/bin/bash
|
|
# pipeline-status.sh — daily health check for the music pipeline.
|
|
#
|
|
# Writes a one-screen summary to ${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log (overwritten daily)
|
|
# and ships it to Telegram via notify-telegram.sh.
|
|
#
|
|
# Reports on:
|
|
# - Sibling service reachability (slskd, navidrome) via HTTP, not docker ps —
|
|
# alembic has no Docker socket access
|
|
# - Today's runs: playlist syncs, Bandcamp sync, manual imports, dedup
|
|
# - Weekly maintenance freshness: strip-mb-tags, strip-watermark-art,
|
|
# scrub-watermark-text, clean-sldl-index, spotify-genre
|
|
# - Daily dedup freshness
|
|
# - Monthly mp3→flac upgrade freshness
|
|
# - Auth & state: Bandcamp cookie expiry, Qobuz token liveness, VPN egress
|
|
# - Storage, Navidrome scan freshness, library size by format
|
|
#
|
|
# Only flags SCRIPT-level failures, not per-file Soulseek errors (those are
|
|
# normal — peers go offline, connections reset).
|
|
|
|
# set -u only, NOT -e/pipefail, on purpose. This script is read-only: it
|
|
# assembles a status digest from ~30 independent probes (grep over logs, beet
|
|
# queries, curl health checks, optional-credential file reads). Each probe is
|
|
# allowed to fail individually and report a warn/skip line; that's the whole
|
|
# design. Under set -e a single missing optional file (e.g. qobuz/app_id) or a
|
|
# no-match grep would abort the entire run and send NO digest at all, which is
|
|
# the opposite of what a health report should do. It writes no library state,
|
|
# so there is nothing to leave half-applied.
|
|
set -u
|
|
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
|
|
OUT=${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log
|
|
TODAY=$(date +%Y%m%d)
|
|
NOW_TS=$(date +%s)
|
|
|
|
NAVIDROME_ENV="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/navidrome/admin.env"
|
|
[ -f "$NAVIDROME_ENV" ] && source "$NAVIDROME_ENV"
|
|
ND_BASE="${ND_BASE:-http://navidrome:4533}"
|
|
ND_USER="${ND_USER:-}"
|
|
ND_PASS="${ND_PASS:-}"
|
|
|
|
# Tally counters for the header banner. Body collected first, then prepended.
|
|
OK=0
|
|
WARN=0
|
|
SKIP=0
|
|
|
|
mark_ok() { OK=$((OK+1)); printf " ✓ %s\n" "$*"; }
|
|
mark_warn() { WARN=$((WARN+1)); printf " ⚠ %s\n" "$*"; }
|
|
mark_skip() { SKIP=$((SKIP+1)); printf " ⊘ %s\n" "$*"; }
|
|
mark_info() { printf " %s\n" "$*"; }
|
|
|
|
section() { printf "\n▎ %s\n" "$*"; }
|
|
|
|
# syslog / `logger` is not present in the container image. Only call it if it
|
|
# exists, so these lines don't spew "logger: command not found" into the job
|
|
# log on every run. Always returns success.
|
|
slog() {
|
|
command -v logger >/dev/null 2>&1 || return 0
|
|
logger -t sldl-pipeline "$@"
|
|
}
|
|
|
|
# Format an age in seconds as "Nh", "Nd", "Nw", "Nmo".
|
|
human_age() {
|
|
local s=$1
|
|
if (( s < 3600 )); then echo "$((s/60))m ago"
|
|
elif (( s < 86400 )); then echo "$((s/3600))h ago"
|
|
elif (( s < 86400*14 )); then echo "$((s/86400))d ago"
|
|
elif (( s < 86400*60 )); then echo "$((s/86400/7))w ago"
|
|
else echo "$((s/86400/30))mo ago"
|
|
fi
|
|
}
|
|
|
|
# Per-playlist log status.
|
|
playlist_status() {
|
|
local log="$1"
|
|
if grep -q "=== Finished playlist run" "$log" 2>/dev/null; then
|
|
# awk picks just the count after "with"; sidesteps the "M3U" digit-3 trap
|
|
# that bites grep -oE '[0-9]+'.
|
|
local m3u
|
|
m3u=$(awk 'match($0, /updated with [0-9]+ tracks/) {
|
|
n=substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", n); last=n
|
|
} END { print last }' "$log")
|
|
echo "OK|${m3u:-0} tracks in M3U"
|
|
elif grep -q "=== Starting playlist run" "$log" 2>/dev/null; then
|
|
echo "WARN|started but never finished"
|
|
else
|
|
echo "WARN|no Starting line"
|
|
fi
|
|
}
|
|
|
|
bandcamp_status() {
|
|
local log="$1"
|
|
if grep -q "=== Bandcamp sync done" "$log" 2>/dev/null; then
|
|
local s
|
|
s=$(grep -oE "\[bandcamp\] === [0-9]+ new[^]]*" "$log" | tail -1 | sed 's/\[bandcamp\] === //; s/ ===$//')
|
|
echo "OK|${s:-completed}"
|
|
else
|
|
echo "WARN|started but never finished"
|
|
fi
|
|
}
|
|
|
|
manual_import_status() {
|
|
local log="$1"
|
|
local n
|
|
# || true, not || echo 0: grep -c already prints "0" on no-match (and exits
|
|
# 1), so || echo 0 produced a two-line "0\n0" that broke the -ge test.
|
|
n=$(grep -c "=== Finished manual import" "$log" 2>/dev/null || true)
|
|
if [[ "${n:-0}" -ge 1 ]]; then
|
|
echo "OK|$n run(s)"
|
|
else
|
|
echo "WARN|started but never finished"
|
|
fi
|
|
}
|
|
|
|
dedup_today_status() {
|
|
local log="$1"
|
|
if grep -q "=== Summary:" "$log" 2>/dev/null; then
|
|
# Format: "=== Summary: N dup groups (parens with commas), N kept, N to delete ==="
|
|
# Strip the wrapper and the parenthesized breakdown so the line stays short.
|
|
local s
|
|
s=$(grep "=== Summary:" "$log" | tail -1 \
|
|
| sed -E 's/.*=== Summary: //; s/ ===.*//; s/ \([^)]*\)//')
|
|
echo "OK|${s:-completed (no summary line)}"
|
|
else
|
|
echo "WARN|started but never finished"
|
|
fi
|
|
}
|
|
|
|
# ---- Build report ----
|
|
{
|
|
echo "__HEADER_PLACEHOLDER__"
|
|
|
|
# 1. Sibling service health
|
|
# alembic deliberately has no Docker socket access (nothing left needs it
|
|
# once sldl/beet run in-process), so these are HTTP reachability checks
|
|
# against each service's own endpoint rather than `docker ps` container
|
|
# state — arguably a better signal anyway (checks the service actually
|
|
# answers, not just that the container process exists).
|
|
section "Services"
|
|
check_http() {
|
|
local name="$1" url="$2"
|
|
if curl -s -o /dev/null --connect-timeout 3 --max-time 6 "$url"; then
|
|
mark_ok "$(printf '%-11s reachable' "$name")"
|
|
else
|
|
mark_warn "$(printf '%-11s unreachable at %s' "$name" "$url")"
|
|
fi
|
|
}
|
|
check_http slskd "http://gluetun:5030/"
|
|
check_http navidrome "http://navidrome:4533/rest/ping.view"
|
|
|
|
# 2. Today's runs
|
|
# Allowlist-based: only Spotify playlists, Bandcamp, manual import, and
|
|
# dedup count as "runs." Everything else (laptop-export, normalize-casing,
|
|
# gen-djmix, gen-vgm, fix-empty-album, fingerprint-index, backfill, replace,
|
|
# convert-m4a, fred-mixes, dj-import-*, weekly tag-hygiene) is either
|
|
# boring/silent maintenance or already covered in the Maintenance section.
|
|
section "Today's runs"
|
|
any_today=0
|
|
newest_dedup_today=$(ls -1t ${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$TODAY-*.log 2>/dev/null | head -1)
|
|
|
|
# Known Spotify playlist names — derive from configs/ directory so adding a
|
|
# new playlist auto-includes it here without editing this script.
|
|
PLAYLIST_NAMES=$(ls ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/*.conf 2>/dev/null \
|
|
| xargs -n1 basename 2>/dev/null \
|
|
| sed 's/\.conf$//' \
|
|
| grep -v '^_' || true)
|
|
|
|
for log in ${ALEMBIC_CONFIG_DIR:-/config}/logs/*-$TODAY.log; do
|
|
[[ -f "$log" ]] || continue
|
|
name=$(basename "$log" .log)
|
|
short=${name%-$TODAY}
|
|
case "$short" in
|
|
manual-import)
|
|
any_today=1
|
|
IFS='|' read -r tag detail < <(manual_import_status "$log") ;;
|
|
bandcamp)
|
|
any_today=1
|
|
IFS='|' read -r tag detail < <(bandcamp_status "$log") ;;
|
|
*)
|
|
# Is this a Spotify playlist log? Check against PLAYLIST_NAMES.
|
|
if echo "$PLAYLIST_NAMES" | grep -qxF "$short"; then
|
|
any_today=1
|
|
IFS='|' read -r tag detail < <(playlist_status "$log")
|
|
else
|
|
continue # not a "run" — silent / belongs in Maintenance / irrelevant
|
|
fi
|
|
;;
|
|
esac
|
|
line=$(printf '%-13s %s' "$short" "$detail")
|
|
if [[ "$tag" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
|
done
|
|
if [[ -n "$newest_dedup_today" ]]; then
|
|
any_today=1
|
|
IFS='|' read -r tag detail < <(dedup_today_status "$newest_dedup_today")
|
|
line=$(printf '%-13s %s' "dedup" "$detail")
|
|
if [[ "$tag" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
|
fi
|
|
[[ "$any_today" -eq 0 ]] && mark_info "(nothing has run yet today)"
|
|
|
|
# 2b. Nightly download tally — count tracks added to beets in the last 24h.
|
|
# Captures the overnight playlist sweep + any daytime Bandcamp/manual imports.
|
|
# Uses beets' `added` timestamp, so it's accurate regardless of how files
|
|
# arrived (Spotify pipeline, Bandcamp, manual). Per-playlist breakdown
|
|
# follows when ≥1 track was added with that grouping today.
|
|
ADDED_TODAY=$(beet ls -f '$grouping' \
|
|
"added:$(date +%Y-%m-%d).." 2>/dev/null || true)
|
|
added_total=$(echo -n "$ADDED_TODAY" | grep -c '^' || true)
|
|
mark_ok "$(printf '%-13s %d new tracks in beets' 'added today' "$added_total")"
|
|
if [[ "$added_total" -gt 0 ]]; then
|
|
# Per-playlist breakdown. Empty $grouping (bandcamp/manual) shown as "(none)".
|
|
echo "$ADDED_TODAY" \
|
|
| awk '{ if ($0 == "") print "(none)"; else print }' \
|
|
| sort | uniq -c | sort -rn \
|
|
| awk '{ g=$2; for(i=3;i<=NF;i++) g=g" "$i; printf " %3d %s\n", $1, g }'
|
|
fi
|
|
|
|
# 3. Maintenance freshness
|
|
# Read from the app's job_runs table (the scheduler's own record) instead of
|
|
# globbing for log files. Jobs run through pipeline_runner write their logs
|
|
# to logs/<job_key>/<timestamp>.log, so the old cron-era filename globs
|
|
# matched nothing for jobs that don't also write their own log, and healthy
|
|
# jobs were reported as "never run yet". The DB also distinguishes lock
|
|
# skips and failures, which a missing log file can't.
|
|
# Limits: weekly tasks should be <9 days old; daily <2; monthly <35.
|
|
section "Maintenance (last run)"
|
|
|
|
declare -A MAINT_LIMIT=(
|
|
[strip-mb-tags]=$((9*86400))
|
|
[strip-watermark-art]=$((9*86400))
|
|
[scrub-watermark-text]=$((9*86400))
|
|
[clean-sldl-index]=$((9*86400))
|
|
[clear-bad-genres]=$((9*86400))
|
|
[spotify-genre]=$((9*86400))
|
|
[dedup-library]=$((2*86400))
|
|
[upgrade-mp3-to-flac]=$((35*86400))
|
|
)
|
|
# Emits one line per task: task|latest_status|latest_age_s|success_age_s|success_log
|
|
# (ages are -1 when there is no such run). Read-only DB open; prints nothing
|
|
# if the DB is missing so every task falls through to "never run yet".
|
|
maint_rows=$(python3 - "${ALEMBIC_CONFIG_DIR:-/config}/alembic.db" <<'PYEOF'
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
|
|
# Digest label -> job_runs.job_key. The scheduled genre refresh records under
|
|
# genre:run (via genre_review_service) and dedup under dedup:scan (via
|
|
# dedup_review_service); everything else is its MAINTENANCE_JOBS key.
|
|
KEYS = [
|
|
("strip-mb-tags", "maintenance:strip_mb_tags"),
|
|
("strip-watermark-art", "maintenance:strip_watermark_art"),
|
|
("scrub-watermark-text", "maintenance:scrub_watermark_text"),
|
|
("clean-sldl-index", "maintenance:clean_sldl_index"),
|
|
("clear-bad-genres", "maintenance:clear_bad_genres"),
|
|
("spotify-genre", "genre:run"),
|
|
("dedup-library", "dedup:scan"),
|
|
("upgrade-mp3-to-flac", "maintenance:upgrade_mp3_to_flac"),
|
|
]
|
|
now = time.time()
|
|
try:
|
|
conn = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
|
|
conn.execute("SELECT 1 FROM job_runs LIMIT 1")
|
|
except sqlite3.Error:
|
|
sys.exit(0)
|
|
for task, key in KEYS:
|
|
latest = conn.execute(
|
|
"SELECT status, started_at FROM job_runs WHERE job_key = ? ORDER BY started_at DESC LIMIT 1",
|
|
(key,),
|
|
).fetchone()
|
|
if latest is None:
|
|
print(f"{task}|none|-1|-1|")
|
|
continue
|
|
success = conn.execute(
|
|
"SELECT started_at, log_path FROM job_runs WHERE job_key = ? AND status = 'success' "
|
|
"ORDER BY started_at DESC LIMIT 1",
|
|
(key,),
|
|
).fetchone()
|
|
s_age = int(now - success[0]) if success else -1
|
|
s_log = (success[1] or "") if success else ""
|
|
print(f"{task}|{latest[0]}|{int(now - latest[1])}|{s_age}|{s_log}")
|
|
PYEOF
|
|
)
|
|
|
|
for task in strip-mb-tags strip-watermark-art scrub-watermark-text clean-sldl-index clear-bad-genres spotify-genre dedup-library upgrade-mp3-to-flac; do
|
|
row=$(grep "^${task}|" <<< "$maint_rows" || true)
|
|
IFS='|' read -r _ latest_status latest_age age_s log <<< "$row"
|
|
if [[ -z "$row" || "$latest_status" == "none" ]]; then
|
|
mark_skip "$(printf '%-22s never run yet' "$task")"
|
|
continue
|
|
fi
|
|
if [[ "$age_s" -lt 0 ]]; then
|
|
# attempted, but never once succeeded -- say what the last attempt did
|
|
case "$latest_status" in
|
|
skipped_lock) note="skipped, lock busy" ;;
|
|
running) note="running now" ;;
|
|
*) note="$latest_status" ;;
|
|
esac
|
|
mark_warn "$(printf '%-22s never succeeded (last attempt %s: %s)' "$task" "$(human_age "$latest_age")" "$note")"
|
|
continue
|
|
fi
|
|
age_h=$(human_age "$age_s")
|
|
snip=""
|
|
if [[ -n "$log" && -f "$log" ]]; then
|
|
case "$task" in
|
|
spotify-genre)
|
|
snip=$(grep -oE "written: [0-9]+, unchanged: [0-9]+" "$log" | tail -1) ;;
|
|
clear-bad-genres)
|
|
snip=$(grep -oE "rewrote: [0-9]+ tracks, blanked: [0-9]+ tracks|(would-clear|cleared): [0-9]+ tracks" "$log" | tail -1) ;;
|
|
clean-sldl-index)
|
|
snip=$(grep -oE "[0-9]+ m3us, total [0-9]+ kept, [0-9]+ dropped" "$log" | tail -1) ;;
|
|
strip-watermark-art)
|
|
snip=$(grep -oE "stripped from [0-9]+ tracks|no images shared" "$log" | tail -1) ;;
|
|
scrub-watermark-text)
|
|
snip=$(grep -oE "stripped [0-9]+ frame\(s\) across [0-9]+ file" "$log" | tail -1) ;;
|
|
strip-mb-tags)
|
|
snip=$(grep -oE "Done\. Log:" "$log" | head -1 | sed 's/Done\. Log:/done/') ;;
|
|
dedup-library)
|
|
snip=$(grep "=== Summary:" "$log" | tail -1 \
|
|
| sed -E 's/.*=== Summary: //; s/ ===.*//; s/ \([^)]*\)//') ;;
|
|
upgrade-mp3-to-flac)
|
|
# The log's "to attempt" count is frozen at the last monthly run, so on
|
|
# its own it looks stale against the live "Library by format" section.
|
|
# Pair it with the current format:mp3 count so the drop (upgrades that
|
|
# have landed since) is visible instead of looking like a mismatch.
|
|
attempted=$(grep -oE "[0-9]+ MP3 tracks to attempt" "$log" | grep -oE "^[0-9]+" | tail -1)
|
|
mp3_now=$(beet ls 'format:mp3' 2>/dev/null | grep -c '^')
|
|
snip="${attempted:-?} attempted · ${mp3_now} MP3 now" ;;
|
|
esac
|
|
fi
|
|
# Age/snippet describe the last SUCCESS; if a newer attempt failed or was
|
|
# lock-skipped, say so rather than hiding it behind the healthy line.
|
|
case "$latest_status" in
|
|
failed) snip="${snip:+$snip, }latest attempt failed" ;;
|
|
skipped_lock) snip="${snip:+$snip, }latest attempt skipped, lock busy" ;;
|
|
esac
|
|
line=$(printf '%-22s %-9s %s' "$task" "$age_h" "${snip:-}")
|
|
if (( age_s > MAINT_LIMIT[$task] )) || [[ "$latest_status" == "failed" ]]; then
|
|
mark_warn "$line"
|
|
else
|
|
mark_ok "$line"
|
|
fi
|
|
done
|
|
|
|
# 4. Auth & state
|
|
section "Auth & state"
|
|
|
|
# Bandcamp cookie
|
|
if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt ]]; then
|
|
exp=$(awk -F'\t' '$6=="identity" {print $5; exit}' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)
|
|
if [[ -n "${exp:-}" && "$exp" =~ ^[0-9]+$ && "$exp" -gt 0 ]]; then
|
|
days=$(( (exp - NOW_TS) / 86400 ))
|
|
line=$(printf '%-22s %d days left' "Bandcamp cookie" "$days")
|
|
if (( days < 14 )); then mark_warn "$line — re-export soon"; else mark_ok "$line"; fi
|
|
else
|
|
mark_warn "Bandcamp cookie could not parse expiry"
|
|
fi
|
|
else
|
|
mark_warn "Bandcamp cookie missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)"
|
|
fi
|
|
|
|
# Qobuz Web Player token (buy-link enrich cascade #2). The token is opaque
|
|
# (no decodable expiry), so probe it live: a cheap authed search returns 200
|
|
# while valid, 401 once it's logged out — at which point enrich-buy-url.py
|
|
# silently drops Qobuz and falls through to iTunes.
|
|
if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token ]]; then
|
|
q_token=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)
|
|
# Guard the app_id read with a file-exists check: `< missing 2>/dev/null`
|
|
# still leaks the shell's own "No such file" redirection error (the redirect
|
|
# is opened before 2>/dev/null takes effect). Most setups have a token but
|
|
# no separate app_id, so this fired on every run.
|
|
q_appid=798273057
|
|
q_appid_file="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/app_id"
|
|
[[ -f "$q_appid_file" ]] && q_appid=$(tr -d '\r\n' < "$q_appid_file")
|
|
q_code=$(curl -s -o /dev/null --connect-timeout 5 --max-time 12 -A "Mozilla/5.0" \
|
|
-H "X-App-Id: $q_appid" -H "X-User-Auth-Token: $q_token" -w '%{http_code}' \
|
|
"https://www.qobuz.com/api.json/0.2/track/search?query=test&limit=1&app_id=$q_appid" 2>/dev/null)
|
|
case "$q_code" in
|
|
200) mark_ok "$(printf '%-22s %s' 'Qobuz token' 'valid (buy-link lookup live)')" ;;
|
|
401) mark_warn "$(printf '%-22s %s' 'Qobuz token' "EXPIRED — re-export X-User-Auth-Token to ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token")" ;;
|
|
*) mark_warn "$(printf '%-22s %s' 'Qobuz token' "check failed (HTTP ${q_code:-none})")" ;;
|
|
esac
|
|
else
|
|
mark_warn "$(printf '%-22s %s' 'Qobuz token' "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)")"
|
|
fi
|
|
|
|
# VPN egress
|
|
# alembic itself rides gluetun's network namespace (network_mode:
|
|
# service:gluetun), so its own outbound traffic IS gluetun's traffic —
|
|
# checking alembic's own egress IP directly is a more direct signal than
|
|
# asking Docker to introspect gluetun's health, and needs no Docker socket.
|
|
egress_ip=$(curl -s --connect-timeout 5 --max-time 10 https://ifconfig.me 2>/dev/null)
|
|
if [[ -n "$egress_ip" ]]; then
|
|
mark_ok "$(printf '%-22s %s' 'VPN egress' "$egress_ip")"
|
|
else
|
|
mark_warn "$(printf '%-22s %s' 'VPN egress' 'could not determine egress IP — VPN may be down')"
|
|
fi
|
|
|
|
# 5. Storage
|
|
section "Storage"
|
|
while read -r mp pct used total; do
|
|
line=$(printf '%-22s %s used (%s of %s)' "$mp" "$pct" "$used" "$total")
|
|
pct_n=${pct%\%}
|
|
if (( pct_n > 90 )); then mark_warn "$line"; else mark_ok "$line"; fi
|
|
done < <(df -h ${MUSIC_DATA_DIR:-/data/music} / 2>/dev/null | awk 'NR>1 && $1!~/tmpfs/ {print $6, $5, $3, $2}')
|
|
|
|
# 6. Navidrome
|
|
section "Navidrome"
|
|
# Password via stdin (p@-), not argv, so it isn't exposed in ps/proc.
|
|
ND_RESP=$(printf '%s' "$ND_PASS" | curl -s --connect-timeout 5 -G "$ND_BASE/rest/getScanStatus.view" \
|
|
--data-urlencode "u=$ND_USER" \
|
|
--data-urlencode "p@-" \
|
|
--data-urlencode 'v=1.16.0' --data-urlencode 'c=status' --data-urlencode 'f=json' 2>/dev/null)
|
|
ND_LINE=$(echo "$ND_RESP" | python3 -c "
|
|
import sys, json
|
|
try:
|
|
d = json.load(sys.stdin)['subsonic-response']
|
|
if d.get('status') == 'ok':
|
|
s = d.get('scanStatus', {})
|
|
cnt = s.get('count', '?')
|
|
cnt_str = f'{cnt:,}' if isinstance(cnt, int) else str(cnt)
|
|
print(f\"OK|last scan {s.get('lastScan','?')[:19]} · {cnt_str} items\")
|
|
else:
|
|
print(f\"WARN|status={d.get('status')}\")
|
|
except Exception as e:
|
|
print(f\"WARN|unreachable: {e}\")
|
|
" 2>/dev/null)
|
|
state=${ND_LINE%%|*}; detail=${ND_LINE##*|}
|
|
line=$(printf '%-22s %s' "Navidrome" "$detail")
|
|
if [[ "$state" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
|
|
|
# 7. Library by format (info-only, no status pill)
|
|
section "Library by format"
|
|
beet ls -f '$format' 2>/dev/null | sort | uniq -c | sort -rn \
|
|
| awk '{printf " %-6s %d tracks\n", $2, $1}'
|
|
|
|
} > "$OUT"
|
|
|
|
# ---- Build header banner and prepend ----
|
|
HOSTNAME_SHORT=$(hostname -s)
|
|
DATE_HUMAN=$(TZ="${TZ:-UTC}" date '+%a %Y-%m-%d %H:%M %Z')
|
|
BANNER_TITLE="🎵 Music pipeline · $HOSTNAME_SHORT · $DATE_HUMAN"
|
|
BANNER_TALLY=" $OK OK · $WARN warn · $SKIP skip"
|
|
sed -i "1c\\
|
|
${BANNER_TITLE}\\
|
|
${BANNER_TALLY}" "$OUT"
|
|
|
|
# Always mirror the one-line summary to syslog so journalctl shows last-run
|
|
# state even if Telegram is broken. Warnings get a separate WARN tag.
|
|
slog "status: $OK ok, $WARN warn, $SKIP skip (see $OUT)"
|
|
if (( WARN > 0 )); then
|
|
slog "WARN: pipeline-status flagged $WARN issue(s) — see $OUT"
|
|
fi
|
|
|
|
# Send to Telegram. If it fails, log the failure to syslog so the absence of
|
|
# a message in the chat has a corresponding journal entry to grep for.
|
|
if [[ -x ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh ]]; then
|
|
if ! ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh < "$OUT" 2>/tmp/tg-err; then
|
|
slog "WARN: Telegram send failed: $(cat /tmp/tg-err 2>/dev/null | head -c 200)"
|
|
rm -f /tmp/tg-err
|
|
fi
|
|
fi
|