alembic/pipeline/lib/pipeline-status.sh

#!/bin/bash
# pipeline-status.sh — daily health check for the music pipeline.
#
# Writes a one-screen summary to ${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log (overwritten daily)
# and ships it to Telegram via notify-telegram.sh.
#
# Reports on:
#   - Sibling service reachability (slskd, navidrome) via HTTP, not docker ps —
#     alembic has no Docker socket access
#   - Today's runs: playlist syncs, Bandcamp sync, manual imports, dedup
#   - Weekly maintenance freshness: strip-mb-tags, strip-watermark-art,
#     scrub-watermark-text, clean-sldl-index, spotify-genre
#   - Daily dedup freshness
#   - Monthly mp3→flac upgrade freshness
#   - Auth & state: Bandcamp cookie expiry, Qobuz token liveness, VPN egress
#   - Storage, Navidrome scan freshness, library size by format
#
# Only flags SCRIPT-level failures, not per-file Soulseek errors (those are
# normal — peers go offline, connections reset).

set -u
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

OUT=${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log
TODAY=$(date +%Y%m%d)
NOW_TS=$(date +%s)

NAVIDROME_ENV="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/navidrome/admin.env"
[ -f "$NAVIDROME_ENV" ] && source "$NAVIDROME_ENV"
ND_BASE="${ND_BASE:-http://navidrome:4533}"
ND_USER="${ND_USER:-}"
ND_PASS="${ND_PASS:-}"

# Tally counters for the header banner. Body collected first, then prepended.
OK=0
WARN=0
SKIP=0

mark_ok()   { OK=$((OK+1));     printf "  ✓ %s\n" "$*"; }
mark_warn() { WARN=$((WARN+1)); printf "  ⚠ %s\n" "$*"; }
mark_skip() { SKIP=$((SKIP+1)); printf "  ⊘ %s\n" "$*"; }
mark_info() {                   printf "    %s\n" "$*"; }

section() { printf "\n▎ %s\n" "$*"; }

# Format an age in seconds as "Nh", "Nd", "Nw", "Nmo".
human_age() {
  local s=$1
  if   (( s < 3600 ));     then echo "$((s/60))m ago"
  elif (( s < 86400 ));    then echo "$((s/3600))h ago"
  elif (( s < 86400*14 )); then echo "$((s/86400))d ago"
  elif (( s < 86400*60 )); then echo "$((s/86400/7))w ago"
  else                          echo "$((s/86400/30))mo ago"
  fi
}

# Newest matching log → "<age_seconds>|<path>", empty if no match.
newest_log() {
  local glob="$1"
  local newest
  newest=$(ls -1t $glob 2>/dev/null | head -1)
  [[ -z "$newest" ]] && return
  local mtime
  mtime=$(stat -c %Y "$newest" 2>/dev/null) || return
  echo "$((NOW_TS - mtime))|$newest"
}

# Per-playlist log status.
playlist_status() {
  local log="$1"
  if grep -q "=== Finished playlist run" "$log" 2>/dev/null; then
    # awk picks just the count after "with"; sidesteps the "M3U" digit-3 trap
    # that bites grep -oE '[0-9]+'.
    local m3u
    m3u=$(awk 'match($0, /updated with [0-9]+ tracks/) {
                 n=substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", n); last=n
               } END { print last }' "$log")
    echo "OK|${m3u:-0} tracks in M3U"
  elif grep -q "=== Starting playlist run" "$log" 2>/dev/null; then
    echo "WARN|started but never finished"
  else
    echo "WARN|no Starting line"
  fi
}

bandcamp_status() {
  local log="$1"
  if grep -q "=== Bandcamp sync done" "$log" 2>/dev/null; then
    local s
    s=$(grep -oE "\[bandcamp\] === [0-9]+ new[^]]*" "$log" | tail -1 | sed 's/\[bandcamp\] === //; s/ ===$//')
    echo "OK|${s:-completed}"
  else
    echo "WARN|started but never finished"
  fi
}

manual_import_status() {
  local log="$1"
  local n
  # || true, not || echo 0: grep -c already prints "0" on no-match (and exits
  # 1), so || echo 0 produced a two-line "0\n0" that broke the -ge test.
  n=$(grep -c "=== Finished manual import" "$log" 2>/dev/null || true)
  if [[ "${n:-0}" -ge 1 ]]; then
    echo "OK|$n run(s)"
  else
    echo "WARN|started but never finished"
  fi
}

dedup_today_status() {
  local log="$1"
  if grep -q "=== Summary:" "$log" 2>/dev/null; then
    # Format: "=== Summary: N dup groups (parens with commas), N kept, N to delete ==="
    # Strip the wrapper and the parenthesized breakdown so the line stays short.
    local s
    s=$(grep "=== Summary:" "$log" | tail -1 \
        | sed -E 's/.*=== Summary: //; s/ ===.*//; s/ \([^)]*\)//')
    echo "OK|${s:-completed (no summary line)}"
  else
    echo "WARN|started but never finished"
  fi
}

# ---- Build report ----
{
  echo "__HEADER_PLACEHOLDER__"

  # 1. Sibling service health
  # alembic deliberately has no Docker socket access (nothing left needs it
  # once sldl/beet run in-process), so these are HTTP reachability checks
  # against each service's own endpoint rather than `docker ps` container
  # state — arguably a better signal anyway (checks the service actually
  # answers, not just that the container process exists).
  section "Services"
  check_http() {
    local name="$1" url="$2"
    if curl -s -o /dev/null --connect-timeout 3 --max-time 6 "$url"; then
      mark_ok "$(printf '%-11s reachable' "$name")"
    else
      mark_warn "$(printf '%-11s unreachable at %s' "$name" "$url")"
    fi
  }
  check_http slskd   "http://gluetun:5030/"
  check_http navidrome "http://navidrome:4533/rest/ping.view"

  # 2. Today's runs
  # Allowlist-based: only Spotify playlists, Bandcamp, manual import, and
  # dedup count as "runs." Everything else (laptop-export, normalize-casing,
  # gen-djmix, gen-vgm, fix-empty-album, fingerprint-index, backfill, replace,
  # convert-m4a, fred-mixes, dj-import-*, weekly tag-hygiene) is either
  # boring/silent maintenance or already covered in the Maintenance section.
  section "Today's runs"
  any_today=0
  newest_dedup_today=$(ls -1t ${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$TODAY-*.log 2>/dev/null | head -1)

  # Known Spotify playlist names — derive from configs/ directory so adding a
  # new playlist auto-includes it here without editing this script.
  PLAYLIST_NAMES=$(ls ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/*.conf 2>/dev/null \
    | xargs -n1 basename 2>/dev/null \
    | sed 's/\.conf$//' \
    | grep -v '^_' || true)

  for log in ${ALEMBIC_CONFIG_DIR:-/config}/logs/*-$TODAY.log; do
    [[ -f "$log" ]] || continue
    name=$(basename "$log" .log)
    short=${name%-$TODAY}
    case "$short" in
      manual-import)
        any_today=1
        IFS='|' read -r tag detail < <(manual_import_status "$log") ;;
      bandcamp)
        any_today=1
        IFS='|' read -r tag detail < <(bandcamp_status "$log") ;;
      *)
        # Is this a Spotify playlist log? Check against PLAYLIST_NAMES.
        if echo "$PLAYLIST_NAMES" | grep -qxF "$short"; then
          any_today=1
          IFS='|' read -r tag detail < <(playlist_status "$log")
        else
          continue  # not a "run" — silent / belongs in Maintenance / irrelevant
        fi
        ;;
    esac
    line=$(printf '%-13s %s' "$short" "$detail")
    if [[ "$tag" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
  done
  if [[ -n "$newest_dedup_today" ]]; then
    any_today=1
    IFS='|' read -r tag detail < <(dedup_today_status "$newest_dedup_today")
    line=$(printf '%-13s %s' "dedup" "$detail")
    if [[ "$tag" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
  fi
  [[ "$any_today" -eq 0 ]] && mark_info "(nothing has run yet today)"

  # 2b. Nightly download tally — count tracks added to beets in the last 24h.
  # Captures the overnight playlist sweep + any daytime Bandcamp/manual imports.
  # Uses beets' `added` timestamp, so it's accurate regardless of how files
  # arrived (Spotify pipeline, Bandcamp, manual). Per-playlist breakdown
  # follows when ≥1 track was added with that grouping today.
  ADDED_TODAY=$(beet ls -f '$grouping' \
    "added:$(date +%Y-%m-%d).." 2>/dev/null || true)
  added_total=$(echo -n "$ADDED_TODAY" | grep -c '^' || true)
  mark_ok "$(printf '%-13s %d new tracks in beets' 'added today' "$added_total")"
  if [[ "$added_total" -gt 0 ]]; then
    # Per-playlist breakdown. Empty $grouping (bandcamp/manual) shown as "(none)".
    echo "$ADDED_TODAY" \
      | awk '{ if ($0 == "") print "(none)"; else print }' \
      | sort | uniq -c | sort -rn \
      | awk '{ g=$2; for(i=3;i<=NF;i++) g=g" "$i; printf "                  %3d  %s\n", $1, g }'
  fi

  # 3. Maintenance freshness
  # Each weekly/monthly task: find its newest log and check age.
  # Limits: weekly tasks should be <9 days old; daily <2; monthly <35.
  section "Maintenance (last run)"

  declare -A MAINT_GLOB=(
    [strip-mb-tags]="${ALEMBIC_CONFIG_DIR:-/config}/logs/mb-strip-*.log"
    [strip-watermark-art]="${ALEMBIC_CONFIG_DIR:-/config}/logs/strip-watermark-*.log"
    [scrub-watermark-text]="${ALEMBIC_CONFIG_DIR:-/config}/logs/scrub-text-*.log"
    [clean-sldl-index]="${ALEMBIC_CONFIG_DIR:-/config}/logs/clean-index-*.log"
    [clear-bad-genres]="${ALEMBIC_CONFIG_DIR:-/config}/logs/clear-bad-genres-*.log"
    [spotify-genre]="${ALEMBIC_CONFIG_DIR:-/config}/logs/spotify-genre-*.log"
    [dedup-library]="${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-*.log"
    [upgrade-mp3-to-flac]="${ALEMBIC_CONFIG_DIR:-/config}/logs/upgrade-mp3-*.log"
  )
  declare -A MAINT_LIMIT=(
    [strip-mb-tags]=$((9*86400))
    [strip-watermark-art]=$((9*86400))
    [scrub-watermark-text]=$((9*86400))
    [clean-sldl-index]=$((9*86400))
    [clear-bad-genres]=$((9*86400))
    [spotify-genre]=$((9*86400))
    [dedup-library]=$((2*86400))
    [upgrade-mp3-to-flac]=$((35*86400))
  )
  for task in strip-mb-tags strip-watermark-art scrub-watermark-text clean-sldl-index clear-bad-genres spotify-genre dedup-library upgrade-mp3-to-flac; do
    res=$(newest_log "${MAINT_GLOB[$task]}")
    if [[ -z "$res" ]]; then
      mark_skip "$(printf '%-22s never run yet' "$task")"
      continue
    fi
    age_s=${res%%|*}
    log=${res##*|}
    age_h=$(human_age "$age_s")
    case "$task" in
      spotify-genre)
        snip=$(grep -oE "written: [0-9]+, unchanged: [0-9]+" "$log" | tail -1) ;;
      clear-bad-genres)
        snip=$(grep -oE "(would-clear|cleared): [0-9]+ tracks" "$log" | tail -1) ;;
      clean-sldl-index)
        snip=$(grep -oE "[0-9]+ m3us, total [0-9]+ kept, [0-9]+ dropped" "$log" | tail -1) ;;
      strip-watermark-art)
        snip=$(grep -oE "stripped from [0-9]+ tracks|no images shared" "$log" | tail -1) ;;
      scrub-watermark-text)
        snip=$(grep -oE "stripped [0-9]+ frame\(s\) across [0-9]+ file" "$log" | tail -1) ;;
      strip-mb-tags)
        snip=$(grep -oE "Done\. Log:" "$log" | head -1 | sed 's/Done\. Log:/done/') ;;
      dedup-library)
        snip=$(grep "=== Summary:" "$log" | tail -1 \
               | sed -E 's/.*=== Summary: //; s/ ===.*//; s/ \([^)]*\)//') ;;
      upgrade-mp3-to-flac)
        # The log's "to attempt" count is frozen at the last monthly run, so on
        # its own it looks stale against the live "Library by format" section.
        # Pair it with the current format:mp3 count so the drop (upgrades that
        # have landed since) is visible instead of looking like a mismatch.
        attempted=$(grep -oE "[0-9]+ MP3 tracks to attempt" "$log" | grep -oE "^[0-9]+" | tail -1)
        mp3_now=$(beet ls 'format:mp3' 2>/dev/null | grep -c '^')
        snip="${attempted:-?} attempted · ${mp3_now} MP3 now" ;;
    esac
    line=$(printf '%-22s %-9s %s' "$task" "$age_h" "${snip:-}")
    if (( age_s > MAINT_LIMIT[$task] )); then
      mark_warn "$line"
    else
      mark_ok "$line"
    fi
  done

  # 4. Auth & state
  section "Auth & state"

  # Bandcamp cookie
  if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt ]]; then
    exp=$(awk -F'\t' '$6=="identity" {print $5; exit}' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)
    if [[ -n "${exp:-}" && "$exp" =~ ^[0-9]+$ && "$exp" -gt 0 ]]; then
      days=$(( (exp - NOW_TS) / 86400 ))
      line=$(printf '%-22s %d days left' "Bandcamp cookie" "$days")
      if (( days < 14 )); then mark_warn "$line — re-export soon"; else mark_ok "$line"; fi
    else
      mark_warn "Bandcamp cookie         could not parse expiry"
    fi
  else
    mark_warn "Bandcamp cookie         missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)"
  fi

  # Qobuz Web Player token (buy-link enrich cascade #2). The token is opaque
  # (no decodable expiry), so probe it live: a cheap authed search returns 200
  # while valid, 401 once it's logged out — at which point enrich-buy-url.py
  # silently drops Qobuz and falls through to iTunes.
  if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token ]]; then
    q_token=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)
    q_appid=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/app_id 2>/dev/null)
    q_appid=${q_appid:-798273057}
    q_code=$(curl -s -o /dev/null --connect-timeout 5 --max-time 12 -A "Mozilla/5.0" \
      -H "X-App-Id: $q_appid" -H "X-User-Auth-Token: $q_token" -w '%{http_code}' \
      "https://www.qobuz.com/api.json/0.2/track/search?query=test&limit=1&app_id=$q_appid" 2>/dev/null)
    case "$q_code" in
      200) mark_ok   "$(printf '%-22s %s' 'Qobuz token' 'valid (buy-link lookup live)')" ;;
      401) mark_warn "$(printf '%-22s %s' 'Qobuz token' "EXPIRED — re-export X-User-Auth-Token to ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token")" ;;
      *)   mark_warn "$(printf '%-22s %s' 'Qobuz token' "check failed (HTTP ${q_code:-none})")" ;;
    esac
  else
    mark_warn "$(printf '%-22s %s' 'Qobuz token' "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)")"
  fi

  # VPN egress
  # alembic itself rides gluetun's network namespace (network_mode:
  # service:gluetun), so its own outbound traffic IS gluetun's traffic —
  # checking alembic's own egress IP directly is a more direct signal than
  # asking Docker to introspect gluetun's health, and needs no Docker socket.
  egress_ip=$(curl -s --connect-timeout 5 --max-time 10 https://ifconfig.me 2>/dev/null)
  if [[ -n "$egress_ip" ]]; then
    mark_ok "$(printf '%-22s %s' 'VPN egress' "$egress_ip")"
  else
    mark_warn "$(printf '%-22s %s' 'VPN egress' 'could not determine egress IP — VPN may be down')"
  fi

  # 5. Storage
  section "Storage"
  while read -r mp pct used total; do
    line=$(printf '%-22s %s used (%s of %s)' "$mp" "$pct" "$used" "$total")
    pct_n=${pct%\%}
    if (( pct_n > 90 )); then mark_warn "$line"; else mark_ok "$line"; fi
  done < <(df -h ${MUSIC_DATA_DIR:-/data/music} / 2>/dev/null | awk 'NR>1 && $1!~/tmpfs/ {print $6, $5, $3, $2}')

  # 6. Navidrome
  section "Navidrome"
  ND_RESP=$(curl -s --connect-timeout 5 -G "$ND_BASE/rest/getScanStatus.view" \
    --data-urlencode "u=$ND_USER" \
    --data-urlencode "p=$ND_PASS" \
    --data-urlencode 'v=1.16.0' --data-urlencode 'c=status' --data-urlencode 'f=json' 2>/dev/null)
  ND_LINE=$(echo "$ND_RESP" | python3 -c "
import sys, json
try:
    d = json.load(sys.stdin)['subsonic-response']
    if d.get('status') == 'ok':
        s = d.get('scanStatus', {})
        cnt = s.get('count', '?')
        cnt_str = f'{cnt:,}' if isinstance(cnt, int) else str(cnt)
        print(f\"OK|last scan {s.get('lastScan','?')[:19]} · {cnt_str} items\")
    else:
        print(f\"WARN|status={d.get('status')}\")
except Exception as e:
    print(f\"WARN|unreachable: {e}\")
" 2>/dev/null)
  state=${ND_LINE%%|*}; detail=${ND_LINE##*|}
  line=$(printf '%-22s %s' "Navidrome" "$detail")
  if [[ "$state" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi

  # 7. Library by format (info-only, no status pill)
  section "Library by format"
  beet ls -f '$format' 2>/dev/null | sort | uniq -c | sort -rn \
    | awk '{printf "    %-6s %d tracks\n", $2, $1}'

} > "$OUT"

# ---- Build header banner and prepend ----
HOSTNAME_SHORT=$(hostname -s)
DATE_HUMAN=$(TZ="${TZ:-UTC}" date '+%a %Y-%m-%d %H:%M %Z')
BANNER_TITLE="🎵 Music pipeline · $HOSTNAME_SHORT · $DATE_HUMAN"
BANNER_TALLY="   $OK OK · $WARN warn · $SKIP skip"
sed -i "1c\\
${BANNER_TITLE}\\
${BANNER_TALLY}" "$OUT"

# Always mirror the one-line summary to syslog so journalctl shows last-run
# state even if Telegram is broken. Warnings get a separate WARN tag.
logger -t sldl-pipeline "status: $OK ok, $WARN warn, $SKIP skip (see $OUT)"
if (( WARN > 0 )); then
  logger -t sldl-pipeline "WARN: pipeline-status flagged $WARN issue(s) — see $OUT"
fi

# Send to Telegram. If it fails, log the failure to syslog so the absence of
# a message in the chat has a corresponding journal entry to grep for.
if [[ -x ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh ]]; then
  if ! ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh < "$OUT" 2>/tmp/tg-err; then
    logger -t sldl-pipeline "WARN: Telegram send failed: $(cat /tmp/tg-err 2>/dev/null | head -c 200)"
    rm -f /tmp/tg-err
  fi
fi