|
|
|
@@ -4,6 +4,19 @@
|
|
|
|
|
# Writes a one-screen summary to ${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log (overwritten daily)
|
|
|
|
|
# and ships it to Telegram via notify-telegram.sh.
|
|
|
|
|
#
|
|
|
|
|
# Formatting note: the digest uses Telegram HTML entities (<b>, <i>, <code>,
|
|
|
|
|
# <blockquote>) — the brand's chrome/dot-badge language translated into what
|
|
|
|
|
# Telegram can actually render (no color, no custom font, so status "dots"
|
|
|
|
|
# become colored circle emoji). Deliberately NO <pre>/monospace column layout:
|
|
|
|
|
# a code block forces fixed-width columns that wrap into unreadable garbage on
|
|
|
|
|
# a phone ("54 tracks in / M3U"). Instead every row is plain reflowing text —
|
|
|
|
|
# a bold label + " — value" — so it wraps cleanly at any screen width. This
|
|
|
|
|
# REQUIRES notify-telegram.sh to be called with --html (parse_mode=HTML) —
|
|
|
|
|
# sent as plain text the tags would show up literally. All free-text going
|
|
|
|
|
# through mark_ok/mark_warn/mark_skip/section is escaped (esc()) for &, <, >
|
|
|
|
|
# so a stray angle bracket in a log line can't break the HTML parse and
|
|
|
|
|
# swallow the whole message.
|
|
|
|
|
#
|
|
|
|
|
# Reports on:
|
|
|
|
|
# - Sibling service reachability (slskd, navidrome) via HTTP, not docker ps —
|
|
|
|
|
# alembic has no Docker socket access
|
|
|
|
@@ -27,7 +40,10 @@
|
|
|
|
|
# the opposite of what a health report should do. It writes no library state,
|
|
|
|
|
# so there is nothing to leave half-applied.
|
|
|
|
|
set -u
|
|
|
|
|
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
|
|
|
# /opt/venv/bin first: `beet` lives in the app venv. Without it every beet
|
|
|
|
|
# probe here ("added today", "Library by format", the mp3-now count) silently
|
|
|
|
|
# came up empty behind its 2>/dev/null.
|
|
|
|
|
PATH=/opt/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
|
|
|
|
|
|
|
|
OUT=${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log
|
|
|
|
|
TODAY=$(date +%Y%m%d)
|
|
|
|
@@ -44,12 +60,39 @@ OK=0
|
|
|
|
|
WARN=0
|
|
|
|
|
SKIP=0
|
|
|
|
|
|
|
|
|
|
mark_ok() { OK=$((OK+1)); printf " ✓ %s\n" "$*"; }
|
|
|
|
|
mark_warn() { WARN=$((WARN+1)); printf " ⚠ %s\n" "$*"; }
|
|
|
|
|
mark_skip() { SKIP=$((SKIP+1)); printf " ⊘ %s\n" "$*"; }
|
|
|
|
|
mark_info() { printf " %s\n" "$*"; }
|
|
|
|
|
# Escapes &, <, > so free text (log snippets, exception messages, filenames)
|
|
|
|
|
# can't be mistaken for an HTML entity by Telegram's parser.
|
|
|
|
|
esc() {
|
|
|
|
|
local s=$1
|
|
|
|
|
s=${s//&/&}
|
|
|
|
|
s=${s//</<}
|
|
|
|
|
s=${s//>/>}
|
|
|
|
|
printf '%s' "$s"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
section() { printf "\n▎ %s\n" "$*"; }
|
|
|
|
|
# One status row: a colored dot, a bold label, and (optionally) " — value".
|
|
|
|
|
# Plain proportional text — NO padding, NO monospace — so it reflows on mobile
|
|
|
|
|
# instead of wrapping mid-column. The dot is the brand's badge-state color,
|
|
|
|
|
# emoji being Telegram's only color channel.
|
|
|
|
|
_row() {
|
|
|
|
|
local dot=$1 label=$2 detail=${3:-}
|
|
|
|
|
if [[ -n "$detail" ]]; then
|
|
|
|
|
printf "%s <b>%s</b> — %s\n" "$dot" "$(esc "$label")" "$(esc "$detail")"
|
|
|
|
|
else
|
|
|
|
|
printf "%s <b>%s</b>\n" "$dot" "$(esc "$label")"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
mark_ok() { OK=$((OK+1)); _row "🟢" "$@"; }
|
|
|
|
|
mark_warn() { WARN=$((WARN+1)); _row "🟠" "$@"; }
|
|
|
|
|
mark_skip() { SKIP=$((SKIP+1)); _row "⚪" "$@"; }
|
|
|
|
|
# Free-text info line (no dot, no counter) — for sub-breakdowns.
|
|
|
|
|
mark_info() { printf " %s\n" "$(esc "$*")"; }
|
|
|
|
|
|
|
|
|
|
# Bold section header (reads like the web app's <h2>). No <pre> — the rows
|
|
|
|
|
# under it are plain reflowing text.
|
|
|
|
|
section() {
|
|
|
|
|
printf "\n<b>▎ %s</b>\n" "$(esc "$*")"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# syslog / `logger` is not present in the container image. Only call it if it
|
|
|
|
|
# exists, so these lines don't spew "logger: command not found" into the job
|
|
|
|
@@ -80,7 +123,7 @@ playlist_status() {
|
|
|
|
|
m3u=$(awk 'match($0, /updated with [0-9]+ tracks/) {
|
|
|
|
|
n=substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", n); last=n
|
|
|
|
|
} END { print last }' "$log")
|
|
|
|
|
echo "OK|${m3u:-0} tracks in M3U"
|
|
|
|
|
echo "OK|${m3u:-0} tracks"
|
|
|
|
|
elif grep -q "=== Starting playlist run" "$log" 2>/dev/null; then
|
|
|
|
|
echo "WARN|started but never finished"
|
|
|
|
|
else
|
|
|
|
@@ -140,12 +183,12 @@ dedup_today_status() {
|
|
|
|
|
check_http() {
|
|
|
|
|
local name="$1" url="$2"
|
|
|
|
|
if curl -s -o /dev/null --connect-timeout 3 --max-time 6 "$url"; then
|
|
|
|
|
mark_ok "$(printf '%-11s reachable' "$name")"
|
|
|
|
|
mark_ok "$name" "reachable"
|
|
|
|
|
else
|
|
|
|
|
mark_warn "$(printf '%-11s unreachable at %s' "$name" "$url")"
|
|
|
|
|
mark_warn "$name" "unreachable at $url"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
check_http slskd "http://gluetun:5030/"
|
|
|
|
|
check_http slskd "http://gluetun:5030/"
|
|
|
|
|
check_http navidrome "http://navidrome:4533/rest/ping.view"
|
|
|
|
|
|
|
|
|
|
# 2. Today's runs
|
|
|
|
@@ -186,14 +229,12 @@ dedup_today_status() {
|
|
|
|
|
fi
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
line=$(printf '%-13s %s' "$short" "$detail")
|
|
|
|
|
if [[ "$tag" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
|
|
|
|
if [[ "$tag" == "OK" ]]; then mark_ok "$short" "$detail"; else mark_warn "$short" "$detail"; fi
|
|
|
|
|
done
|
|
|
|
|
if [[ -n "$newest_dedup_today" ]]; then
|
|
|
|
|
any_today=1
|
|
|
|
|
IFS='|' read -r tag detail < <(dedup_today_status "$newest_dedup_today")
|
|
|
|
|
line=$(printf '%-13s %s' "dedup" "$detail")
|
|
|
|
|
if [[ "$tag" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
|
|
|
|
if [[ "$tag" == "OK" ]]; then mark_ok "dedup" "$detail"; else mark_warn "dedup" "$detail"; fi
|
|
|
|
|
fi
|
|
|
|
|
[[ "$any_today" -eq 0 ]] && mark_info "(nothing has run yet today)"
|
|
|
|
|
|
|
|
|
@@ -205,13 +246,16 @@ dedup_today_status() {
|
|
|
|
|
ADDED_TODAY=$(beet ls -f '$grouping' \
|
|
|
|
|
"added:$(date +%Y-%m-%d).." 2>/dev/null || true)
|
|
|
|
|
added_total=$(echo -n "$ADDED_TODAY" | grep -c '^' || true)
|
|
|
|
|
mark_ok "$(printf '%-13s %d new tracks in beets' 'added today' "$added_total")"
|
|
|
|
|
mark_ok "added today" "$added_total new tracks in beets"
|
|
|
|
|
if [[ "$added_total" -gt 0 ]]; then
|
|
|
|
|
# Per-playlist breakdown. Empty $grouping (bandcamp/manual) shown as "(none)".
|
|
|
|
|
# Piped through the same &/</> escaping as esc(), since this prints
|
|
|
|
|
# straight into the message without going through _row.
|
|
|
|
|
echo "$ADDED_TODAY" \
|
|
|
|
|
| awk '{ if ($0 == "") print "(none)"; else print }' \
|
|
|
|
|
| sort | uniq -c | sort -rn \
|
|
|
|
|
| awk '{ g=$2; for(i=3;i<=NF;i++) g=g" "$i; printf " %3d %s\n", $1, g }'
|
|
|
|
|
| awk '{ c=$1; g=$2; for(i=3;i<=NF;i++) g=g" "$i; printf " • %s (%d)\n", g, c }' \
|
|
|
|
|
| sed 's/&/\&/g; s/</\</g; s/>/\>/g'
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# 3. Maintenance freshness
|
|
|
|
@@ -284,7 +328,7 @@ PYEOF
|
|
|
|
|
row=$(grep "^${task}|" <<< "$maint_rows" || true)
|
|
|
|
|
IFS='|' read -r _ latest_status latest_age age_s log <<< "$row"
|
|
|
|
|
if [[ -z "$row" || "$latest_status" == "none" ]]; then
|
|
|
|
|
mark_skip "$(printf '%-22s never run yet' "$task")"
|
|
|
|
|
mark_skip "$task" "never run yet"
|
|
|
|
|
continue
|
|
|
|
|
fi
|
|
|
|
|
if [[ "$age_s" -lt 0 ]]; then
|
|
|
|
@@ -294,7 +338,7 @@ PYEOF
|
|
|
|
|
running) note="running now" ;;
|
|
|
|
|
*) note="$latest_status" ;;
|
|
|
|
|
esac
|
|
|
|
|
mark_warn "$(printf '%-22s never succeeded (last attempt %s: %s)' "$task" "$(human_age "$latest_age")" "$note")"
|
|
|
|
|
mark_warn "$task" "never succeeded (last attempt $(human_age "$latest_age"): $note)"
|
|
|
|
|
continue
|
|
|
|
|
fi
|
|
|
|
|
age_h=$(human_age "$age_s")
|
|
|
|
@@ -332,11 +376,12 @@ PYEOF
|
|
|
|
|
failed) snip="${snip:+$snip, }latest attempt failed" ;;
|
|
|
|
|
skipped_lock) snip="${snip:+$snip, }latest attempt skipped, lock busy" ;;
|
|
|
|
|
esac
|
|
|
|
|
line=$(printf '%-22s %-9s %s' "$task" "$age_h" "${snip:-}")
|
|
|
|
|
detail="$age_h"
|
|
|
|
|
[[ -n "$snip" ]] && detail="$age_h · $snip"
|
|
|
|
|
if (( age_s > MAINT_LIMIT[$task] )) || [[ "$latest_status" == "failed" ]]; then
|
|
|
|
|
mark_warn "$line"
|
|
|
|
|
mark_warn "$task" "$detail"
|
|
|
|
|
else
|
|
|
|
|
mark_ok "$line"
|
|
|
|
|
mark_ok "$task" "$detail"
|
|
|
|
|
fi
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
@@ -348,13 +393,16 @@ PYEOF
|
|
|
|
|
exp=$(awk -F'\t' '$6=="identity" {print $5; exit}' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)
|
|
|
|
|
if [[ -n "${exp:-}" && "$exp" =~ ^[0-9]+$ && "$exp" -gt 0 ]]; then
|
|
|
|
|
days=$(( (exp - NOW_TS) / 86400 ))
|
|
|
|
|
line=$(printf '%-22s %d days left' "Bandcamp cookie" "$days")
|
|
|
|
|
if (( days < 14 )); then mark_warn "$line — re-export soon"; else mark_ok "$line"; fi
|
|
|
|
|
if (( days < 14 )); then
|
|
|
|
|
mark_warn "Bandcamp cookie" "$days days left — re-export soon"
|
|
|
|
|
else
|
|
|
|
|
mark_ok "Bandcamp cookie" "$days days left"
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
mark_warn "Bandcamp cookie could not parse expiry"
|
|
|
|
|
mark_warn "Bandcamp cookie" "could not parse expiry"
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
mark_warn "Bandcamp cookie missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)"
|
|
|
|
|
mark_warn "Bandcamp cookie" "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Qobuz Web Player token (buy-link enrich cascade #2). The token is opaque
|
|
|
|
@@ -374,12 +422,12 @@ PYEOF
|
|
|
|
|
-H "X-App-Id: $q_appid" -H "X-User-Auth-Token: $q_token" -w '%{http_code}' \
|
|
|
|
|
"https://www.qobuz.com/api.json/0.2/track/search?query=test&limit=1&app_id=$q_appid" 2>/dev/null)
|
|
|
|
|
case "$q_code" in
|
|
|
|
|
200) mark_ok "$(printf '%-22s %s' 'Qobuz token' 'valid (buy-link lookup live)')" ;;
|
|
|
|
|
401) mark_warn "$(printf '%-22s %s' 'Qobuz token' "EXPIRED — re-export X-User-Auth-Token to ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token")" ;;
|
|
|
|
|
*) mark_warn "$(printf '%-22s %s' 'Qobuz token' "check failed (HTTP ${q_code:-none})")" ;;
|
|
|
|
|
200) mark_ok "Qobuz token" "valid (buy-link lookup live)" ;;
|
|
|
|
|
401) mark_warn "Qobuz token" "EXPIRED — re-export X-User-Auth-Token to ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token" ;;
|
|
|
|
|
*) mark_warn "Qobuz token" "check failed (HTTP ${q_code:-none})" ;;
|
|
|
|
|
esac
|
|
|
|
|
else
|
|
|
|
|
mark_warn "$(printf '%-22s %s' 'Qobuz token' "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)")"
|
|
|
|
|
mark_warn "Qobuz token" "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# VPN egress
|
|
|
|
@@ -389,17 +437,20 @@ PYEOF
|
|
|
|
|
# asking Docker to introspect gluetun's health, and needs no Docker socket.
|
|
|
|
|
egress_ip=$(curl -s --connect-timeout 5 --max-time 10 https://ifconfig.me 2>/dev/null)
|
|
|
|
|
if [[ -n "$egress_ip" ]]; then
|
|
|
|
|
mark_ok "$(printf '%-22s %s' 'VPN egress' "$egress_ip")"
|
|
|
|
|
mark_ok "VPN egress" "$egress_ip"
|
|
|
|
|
else
|
|
|
|
|
mark_warn "$(printf '%-22s %s' 'VPN egress' 'could not determine egress IP — VPN may be down')"
|
|
|
|
|
mark_warn "VPN egress" "could not determine egress IP — VPN may be down"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# 5. Storage
|
|
|
|
|
section "Storage"
|
|
|
|
|
while read -r mp pct used total; do
|
|
|
|
|
line=$(printf '%-22s %s used (%s of %s)' "$mp" "$pct" "$used" "$total")
|
|
|
|
|
pct_n=${pct%\%}
|
|
|
|
|
if (( pct_n > 90 )); then mark_warn "$line"; else mark_ok "$line"; fi
|
|
|
|
|
if (( pct_n > 90 )); then
|
|
|
|
|
mark_warn "$mp" "$pct used ($used of $total)"
|
|
|
|
|
else
|
|
|
|
|
mark_ok "$mp" "$pct used ($used of $total)"
|
|
|
|
|
fi
|
|
|
|
|
done < <(df -h ${MUSIC_DATA_DIR:-/data/music} / 2>/dev/null | awk 'NR>1 && $1!~/tmpfs/ {print $6, $5, $3, $2}')
|
|
|
|
|
|
|
|
|
|
# 6. Navidrome
|
|
|
|
@@ -424,24 +475,40 @@ except Exception as e:
|
|
|
|
|
print(f\"WARN|unreachable: {e}\")
|
|
|
|
|
" 2>/dev/null)
|
|
|
|
|
state=${ND_LINE%%|*}; detail=${ND_LINE##*|}
|
|
|
|
|
line=$(printf '%-22s %s' "Navidrome" "$detail")
|
|
|
|
|
if [[ "$state" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
|
|
|
|
if [[ "$state" == "OK" ]]; then mark_ok "Navidrome" "$detail"; else mark_warn "Navidrome" "$detail"; fi
|
|
|
|
|
|
|
|
|
|
# 7. Library by format (info-only, no status pill)
|
|
|
|
|
# 7. Library by format (info-only, no status dot)
|
|
|
|
|
section "Library by format"
|
|
|
|
|
beet ls -f '$format' 2>/dev/null | sort | uniq -c | sort -rn \
|
|
|
|
|
| awk '{printf " %-6s %d tracks\n", $2, $1}'
|
|
|
|
|
| awk '{printf " • %s — %d tracks\n", $2, $1}' \
|
|
|
|
|
| sed 's/&/\&/g; s/</\</g; s/>/\>/g'
|
|
|
|
|
|
|
|
|
|
} > "$OUT"
|
|
|
|
|
|
|
|
|
|
# ---- Build header banner and prepend ----
|
|
|
|
|
# Bold brand mark (⚗️ is literally the "ALEMBIC" unicode glyph — no generic
|
|
|
|
|
# music-note stand-in needed), a hostname chip in <code>, and an italic tally
|
|
|
|
|
# that reuses the same 🟢/🟠/⚪ dots as the body. When something needs
|
|
|
|
|
# attention, a <blockquote> callout mirrors the web app's .notice.warning
|
|
|
|
|
# panel; an all-clear run gets the calm .notice.success equivalent instead.
|
|
|
|
|
HOSTNAME_SHORT=$(hostname -s)
|
|
|
|
|
DATE_HUMAN=$(TZ="${TZ:-UTC}" date '+%a %Y-%m-%d %H:%M %Z')
|
|
|
|
|
BANNER_TITLE="🎵 Music pipeline · $HOSTNAME_SHORT · $DATE_HUMAN"
|
|
|
|
|
BANNER_TALLY=" $OK OK · $WARN warn · $SKIP skip"
|
|
|
|
|
sed -i "1c\\
|
|
|
|
|
${BANNER_TITLE}\\
|
|
|
|
|
${BANNER_TALLY}" "$OUT"
|
|
|
|
|
HEADER="⚗️ <b>Alembic</b> · music pipeline · <code>${HOSTNAME_SHORT}</code> · ${DATE_HUMAN}
|
|
|
|
|
<i>🟢 ${OK} OK 🟠 ${WARN} warn ⚪ ${SKIP} skip</i>"
|
|
|
|
|
if (( WARN > 0 )); then
|
|
|
|
|
HEADER="${HEADER}
|
|
|
|
|
<blockquote>🟠 <b>${WARN} issue(s)</b> need attention — see below</blockquote>"
|
|
|
|
|
else
|
|
|
|
|
HEADER="${HEADER}
|
|
|
|
|
<blockquote>🟢 All clear — nothing needs attention.</blockquote>"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Swap the __HEADER_PLACEHOLDER__ line for the (2-3 line) banner above. A
|
|
|
|
|
# straight prepend, not sed 1c — the banner's line count varies with the
|
|
|
|
|
# blockquote, which sed's `c` command can't take as a variable easily.
|
|
|
|
|
tail -n +2 "$OUT" > "${OUT}.body"
|
|
|
|
|
{ printf '%s\n' "$HEADER"; cat "${OUT}.body"; } > "$OUT"
|
|
|
|
|
rm -f "${OUT}.body"
|
|
|
|
|
|
|
|
|
|
# Always mirror the one-line summary to syslog so journalctl shows last-run
|
|
|
|
|
# state even if Telegram is broken. Warnings get a separate WARN tag.
|
|
|
|
@@ -450,10 +517,12 @@ if (( WARN > 0 )); then
|
|
|
|
|
slog "WARN: pipeline-status flagged $WARN issue(s) — see $OUT"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Send to Telegram. If it fails, log the failure to syslog so the absence of
|
|
|
|
|
# a message in the chat has a corresponding journal entry to grep for.
|
|
|
|
|
# Send to Telegram with --html: the digest is Telegram-HTML (see header
|
|
|
|
|
# comment), so notify-telegram.sh must send it with parse_mode=HTML. If the
|
|
|
|
|
# send fails, log to syslog so the absence of a message in the chat has a
|
|
|
|
|
# corresponding journal entry to grep for.
|
|
|
|
|
if [[ -x ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh ]]; then
|
|
|
|
|
if ! ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh < "$OUT" 2>/tmp/tg-err; then
|
|
|
|
|
if ! ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh --html < "$OUT" 2>/tmp/tg-err; then
|
|
|
|
|
slog "WARN: Telegram send failed: $(cat /tmp/tg-err 2>/dev/null | head -c 200)"
|
|
|
|
|
rm -f /tmp/tg-err
|
|
|
|
|
fi
|
|
|
|
|
andrew