alembic/pipeline/lib/pipeline-status.sh

#!/bin/bash
# pipeline-status.sh — daily health check for the music pipeline.
#
# Writes a one-screen summary to ${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log (overwritten daily)
# and ships it to Telegram via notify-telegram.sh.
#
# Formatting note: the digest uses Telegram HTML entities (<b>, <i>, <code>,
# <blockquote>) — the brand's chrome/dot-badge language translated into what
# Telegram can actually render (no color, no custom font, so status "dots"
# become colored circle emoji). Deliberately NO <pre>/monospace column layout:
# a code block forces fixed-width columns that wrap into unreadable garbage on
# a phone ("54 tracks in / M3U"). Instead every row is plain reflowing text —
# a bold label + " — value" — so it wraps cleanly at any screen width. This
# REQUIRES notify-telegram.sh to be called with --html (parse_mode=HTML) —
# sent as plain text the tags would show up literally. All free-text going
# through mark_ok/mark_warn/mark_skip/section is escaped (esc()) for &, <, >
# so a stray angle bracket in a log line can't break the HTML parse and
# swallow the whole message.
#
# Reports on:
#   - Sibling service reachability (navidrome) via HTTP, not docker ps —
#     alembic has no Docker socket access. slskd is deliberately NOT checked:
#     it is not part of the pipeline (sldl is its own Soulseek client) — it
#     runs on the host purely as the user's own file-sharing presence.
#   - Today's runs: playlist syncs, Bandcamp sync, manual imports, dedup
#   - Weekly maintenance freshness: strip-mb-tags, strip-watermark-art,
#     scrub-watermark-text, clean-sldl-index, spotify-genre
#   - Daily dedup freshness
#   - Monthly mp3→flac upgrade freshness
#   - Auth & state: Bandcamp cookie expiry, Qobuz token liveness, VPN egress
#   - Storage, Navidrome scan freshness, library size by format
#
# Only flags SCRIPT-level failures, not per-file Soulseek errors (those are
# normal — peers go offline, connections reset).

# set -u only, NOT -e/pipefail, on purpose. This script is read-only: it
# assembles a status digest from ~30 independent probes (grep over logs, beet
# queries, curl health checks, optional-credential file reads). Each probe is
# allowed to fail individually and report a warn/skip line; that's the whole
# design. Under set -e a single missing optional file (e.g. qobuz/app_id) or a
# no-match grep would abort the entire run and send NO digest at all, which is
# the opposite of what a health report should do. It writes no library state,
# so there is nothing to leave half-applied.
set -u
# /opt/venv/bin first: `beet` lives in the app venv. Without it every beet
# probe here ("added today", "Library by format", the mp3-now count) silently
# came up empty behind its 2>/dev/null.
PATH=/opt/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

OUT=${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log
TODAY=$(date +%Y%m%d)
NOW_TS=$(date +%s)

NAVIDROME_ENV="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/navidrome/admin.env"
[ -f "$NAVIDROME_ENV" ] && source "$NAVIDROME_ENV"
ND_BASE="${ND_BASE:-http://navidrome:4533}"
ND_USER="${ND_USER:-}"
ND_PASS="${ND_PASS:-}"

# Tally counters for the header banner. Body collected first, then prepended.
OK=0
WARN=0
SKIP=0

# Escapes &, <, > so free text (log snippets, exception messages, filenames)
# can't be mistaken for an HTML entity by Telegram's parser.
esc() {
  local s=$1
  s=${s//&/&amp;}
  s=${s//</&lt;}
  s=${s//>/&gt;}
  printf '%s' "$s"
}

# One status row: a colored dot, a bold label, and (optionally) " — value".
# Plain proportional text — NO padding, NO monospace — so it reflows on mobile
# instead of wrapping mid-column. The dot is the brand's badge-state color,
# emoji being Telegram's only color channel.
_row() {
  local dot=$1 label=$2 detail=${3:-}
  if [[ -n "$detail" ]]; then
    printf "%s <b>%s</b> — %s\n" "$dot" "$(esc "$label")" "$(esc "$detail")"
  else
    printf "%s <b>%s</b>\n" "$dot" "$(esc "$label")"
  fi
}
mark_ok()   { OK=$((OK+1));     _row "🟢" "$@"; }
mark_warn() { WARN=$((WARN+1)); _row "🟠" "$@"; }
mark_skip() { SKIP=$((SKIP+1)); _row "⚪" "$@"; }
# Free-text info line (no dot, no counter) — for sub-breakdowns.
mark_info() { printf "   %s\n" "$(esc "$*")"; }

# Bold section header (reads like the web app's <h2>). No <pre> — the rows
# under it are plain reflowing text.
section() {
  printf "\n<b>▎ %s</b>\n" "$(esc "$*")"
}

# syslog / `logger` is not present in the container image. Only call it if it
# exists, so these lines don't spew "logger: command not found" into the job
# log on every run. Always returns success.
slog() {
  command -v logger >/dev/null 2>&1 || return 0
  logger -t sldl-pipeline "$@"
}

# Format an age in seconds as "Nh", "Nd", "Nw", "Nmo".
human_age() {
  local s=$1
  if   (( s < 3600 ));     then echo "$((s/60))m ago"
  elif (( s < 86400 ));    then echo "$((s/3600))h ago"
  elif (( s < 86400*14 )); then echo "$((s/86400))d ago"
  elif (( s < 86400*60 )); then echo "$((s/86400/7))w ago"
  else                          echo "$((s/86400/30))mo ago"
  fi
}

# Per-playlist log status.
playlist_status() {
  local log="$1"
  if grep -q "=== Finished playlist run" "$log" 2>/dev/null; then
    # awk picks just the count after "with"; sidesteps the "M3U" digit-3 trap
    # that bites grep -oE '[0-9]+'.
    local m3u
    m3u=$(awk 'match($0, /updated with [0-9]+ tracks/) {
                 n=substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", n); last=n
               } END { print last }' "$log")
    echo "OK|${m3u:-0} tracks"
  elif grep -q "=== Starting playlist run" "$log" 2>/dev/null; then
    echo "WARN|started but never finished"
  else
    echo "WARN|no Starting line"
  fi
}

bandcamp_status() {
  local log="$1"
  if grep -q "=== Bandcamp sync done" "$log" 2>/dev/null; then
    local s
    s=$(grep -oE "\[bandcamp\] === [0-9]+ new[^]]*" "$log" | tail -1 | sed 's/\[bandcamp\] === //; s/ ===$//')
    echo "OK|${s:-completed}"
  else
    echo "WARN|started but never finished"
  fi
}

manual_import_status() {
  local log="$1"
  local n
  # || true, not || echo 0: grep -c already prints "0" on no-match (and exits
  # 1), so || echo 0 produced a two-line "0\n0" that broke the -ge test.
  n=$(grep -c "=== Finished manual import" "$log" 2>/dev/null || true)
  if [[ "${n:-0}" -ge 1 ]]; then
    echo "OK|$n run(s)"
  else
    echo "WARN|started but never finished"
  fi
}

dedup_today_status() {
  local log="$1"
  if grep -q "=== Summary:" "$log" 2>/dev/null; then
    # Format: "=== Summary: N dup groups (parens with commas), N kept, N to delete ==="
    # Strip the wrapper and the parenthesized breakdown so the line stays short.
    local s
    s=$(grep "=== Summary:" "$log" | tail -1 \
        | sed -E 's/.*=== Summary: //; s/ ===.*//; s/ \([^)]*\)//')
    echo "OK|${s:-completed (no summary line)}"
  else
    echo "WARN|started but never finished"
  fi
}

# ---- Build report ----
{
  echo "__HEADER_PLACEHOLDER__"

  # 1. Sibling service health
  # alembic deliberately has no Docker socket access (nothing left needs it
  # once sldl/beet run in-process), so these are HTTP reachability checks
  # against each service's own endpoint rather than `docker ps` container
  # state — arguably a better signal anyway (checks the service actually
  # answers, not just that the container process exists).
  section "Services"
  check_http() {
    local name="$1" url="$2"
    if curl -s -o /dev/null --connect-timeout 3 --max-time 6 "$url"; then
      mark_ok "$name" "reachable"
    else
      mark_warn "$name" "unreachable at $url"
    fi
  }
  check_http navidrome "http://navidrome:4533/rest/ping.view"

  # 2. Today's runs
  # Allowlist-based: only Spotify playlists, Bandcamp, manual import, and
  # dedup count as "runs." Everything else (laptop-export, normalize-casing,
  # gen-djmix, gen-vgm, fix-empty-album, fingerprint-index, backfill, replace,
  # convert-m4a, fred-mixes, dj-import-*, weekly tag-hygiene) is either
  # boring/silent maintenance or already covered in the Maintenance section.
  section "Today's runs"
  any_today=0
  newest_dedup_today=$(ls -1t ${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$TODAY-*.log 2>/dev/null | head -1)

  # Known Spotify playlist names — derive from configs/ directory so adding a
  # new playlist auto-includes it here without editing this script.
  PLAYLIST_NAMES=$(ls ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/*.conf 2>/dev/null \
    | xargs -n1 basename 2>/dev/null \
    | sed 's/\.conf$//' \
    | grep -v '^_' || true)

  for log in ${ALEMBIC_CONFIG_DIR:-/config}/logs/*-$TODAY.log; do
    [[ -f "$log" ]] || continue
    name=$(basename "$log" .log)
    short=${name%-$TODAY}
    case "$short" in
      manual-import)
        any_today=1
        IFS='|' read -r tag detail < <(manual_import_status "$log") ;;
      bandcamp)
        any_today=1
        IFS='|' read -r tag detail < <(bandcamp_status "$log") ;;
      *)
        # Is this a Spotify playlist log? Check against PLAYLIST_NAMES.
        if echo "$PLAYLIST_NAMES" | grep -qxF "$short"; then
          any_today=1
          IFS='|' read -r tag detail < <(playlist_status "$log")
        else
          continue  # not a "run" — silent / belongs in Maintenance / irrelevant
        fi
        ;;
    esac
    if [[ "$tag" == "OK" ]]; then mark_ok "$short" "$detail"; else mark_warn "$short" "$detail"; fi
  done
  if [[ -n "$newest_dedup_today" ]]; then
    any_today=1
    IFS='|' read -r tag detail < <(dedup_today_status "$newest_dedup_today")
    if [[ "$tag" == "OK" ]]; then mark_ok "dedup" "$detail"; else mark_warn "dedup" "$detail"; fi
  fi
  [[ "$any_today" -eq 0 ]] && mark_info "(nothing has run yet today)"

  # 2b. Nightly download tally — count tracks added to beets in the last 24h.
  # Captures the overnight playlist sweep + any daytime Bandcamp/manual imports.
  # Uses beets' `added` timestamp, so it's accurate regardless of how files
  # arrived (Spotify pipeline, Bandcamp, manual). Per-playlist breakdown
  # follows when ≥1 track was added with that grouping today.
  ADDED_TODAY=$(beet ls -f '$grouping' \
    "added:$(date +%Y-%m-%d).." 2>/dev/null || true)
  added_total=$(echo -n "$ADDED_TODAY" | grep -c '^' || true)
  mark_ok "added today" "$added_total new tracks in beets"
  if [[ "$added_total" -gt 0 ]]; then
    # Per-playlist breakdown. Empty $grouping (bandcamp/manual) shown as "(none)".
    # Piped through the same &/</> escaping as esc(), since this prints
    # straight into the message without going through _row.
    echo "$ADDED_TODAY" \
      | awk '{ if ($0 == "") print "(none)"; else print }' \
      | sort | uniq -c | sort -rn \
      | awk '{ c=$1; g=$2; for(i=3;i<=NF;i++) g=g" "$i; printf "   • %s (%d)\n", g, c }' \
      | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g'
  fi

  # 3. Maintenance freshness
  # Read from the app's job_runs table (the scheduler's own record) instead of
  # globbing for log files. Jobs run through pipeline_runner write their logs
  # to logs/<job_key>/<timestamp>.log, so the old cron-era filename globs
  # matched nothing for jobs that don't also write their own log, and healthy
  # jobs were reported as "never run yet". The DB also distinguishes lock
  # skips and failures, which a missing log file can't.
  # Limits: weekly tasks should be <9 days old; daily <2; monthly <35.
  section "Maintenance (last run)"

  declare -A MAINT_LIMIT=(
    [strip-mb-tags]=$((9*86400))
    [strip-watermark-art]=$((9*86400))
    [scrub-watermark-text]=$((9*86400))
    [clean-sldl-index]=$((9*86400))
    [clear-bad-genres]=$((9*86400))
    [spotify-genre]=$((9*86400))
    [dedup-library]=$((2*86400))
    [upgrade-mp3-to-flac]=$((35*86400))
  )
  # Emits one line per task: task|latest_status|latest_age_s|success_age_s|success_log
  # (ages are -1 when there is no such run). Read-only DB open; prints nothing
  # if the DB is missing so every task falls through to "never run yet".
  maint_rows=$(python3 - "${ALEMBIC_CONFIG_DIR:-/config}/alembic.db" <<'PYEOF'
import sqlite3
import sys
import time

# Digest label -> job_runs.job_key. The scheduled genre refresh records under
# genre:run (via genre_review_service) and dedup under dedup:scan (via
# dedup_review_service); everything else is its MAINTENANCE_JOBS key.
KEYS = [
    ("strip-mb-tags", "maintenance:strip_mb_tags"),
    ("strip-watermark-art", "maintenance:strip_watermark_art"),
    ("scrub-watermark-text", "maintenance:scrub_watermark_text"),
    ("clean-sldl-index", "maintenance:clean_sldl_index"),
    ("clear-bad-genres", "maintenance:clear_bad_genres"),
    ("spotify-genre", "genre:run"),
    ("dedup-library", "dedup:scan"),
    ("upgrade-mp3-to-flac", "maintenance:upgrade_mp3_to_flac"),
]
now = time.time()
try:
    conn = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
    conn.execute("SELECT 1 FROM job_runs LIMIT 1")
except sqlite3.Error:
    sys.exit(0)
for task, key in KEYS:
    latest = conn.execute(
        "SELECT status, started_at FROM job_runs WHERE job_key = ? ORDER BY started_at DESC LIMIT 1",
        (key,),
    ).fetchone()
    if latest is None:
        print(f"{task}|none|-1|-1|")
        continue
    success = conn.execute(
        "SELECT started_at, log_path FROM job_runs WHERE job_key = ? AND status = 'success' "
        "ORDER BY started_at DESC LIMIT 1",
        (key,),
    ).fetchone()
    s_age = int(now - success[0]) if success else -1
    s_log = (success[1] or "") if success else ""
    print(f"{task}|{latest[0]}|{int(now - latest[1])}|{s_age}|{s_log}")
PYEOF
)

  for task in strip-mb-tags strip-watermark-art scrub-watermark-text clean-sldl-index clear-bad-genres spotify-genre dedup-library upgrade-mp3-to-flac; do
    row=$(grep "^${task}|" <<< "$maint_rows" || true)
    IFS='|' read -r _ latest_status latest_age age_s log <<< "$row"
    if [[ -z "$row" || "$latest_status" == "none" ]]; then
      mark_skip "$task" "never run yet"
      continue
    fi
    if [[ "$age_s" -lt 0 ]]; then
      # attempted, but never once succeeded -- say what the last attempt did
      case "$latest_status" in
        skipped_lock) note="skipped, lock busy" ;;
        running)      note="running now" ;;
        *)            note="$latest_status" ;;
      esac
      mark_warn "$task" "never succeeded (last attempt $(human_age "$latest_age"): $note)"
      continue
    fi
    age_h=$(human_age "$age_s")
    snip=""
    if [[ -n "$log" && -f "$log" ]]; then
      case "$task" in
        spotify-genre)
          snip=$(grep -oE "written: [0-9]+, unchanged: [0-9]+" "$log" | tail -1) ;;
        clear-bad-genres)
          snip=$(grep -oE "rewrote: [0-9]+ tracks, blanked: [0-9]+ tracks|(would-clear|cleared): [0-9]+ tracks" "$log" | tail -1) ;;
        clean-sldl-index)
          snip=$(grep -oE "[0-9]+ m3us, total [0-9]+ kept, [0-9]+ dropped" "$log" | tail -1) ;;
        strip-watermark-art)
          snip=$(grep -oE "stripped from [0-9]+ tracks|no images shared" "$log" | tail -1) ;;
        scrub-watermark-text)
          snip=$(grep -oE "stripped [0-9]+ frame\(s\) across [0-9]+ file" "$log" | tail -1) ;;
        strip-mb-tags)
          snip=$(grep -oE "Done\. Log:" "$log" | head -1 | sed 's/Done\. Log:/done/') ;;
        dedup-library)
          snip=$(grep "=== Summary:" "$log" | tail -1 \
                 | sed -E 's/.*=== Summary: //; s/ ===.*//; s/ \([^)]*\)//') ;;
        upgrade-mp3-to-flac)
          # The log's "to attempt" count is frozen at the last monthly run, so on
          # its own it looks stale against the live "Library by format" section.
          # Pair it with the current format:mp3 count so the drop (upgrades that
          # have landed since) is visible instead of looking like a mismatch.
          attempted=$(grep -oE "[0-9]+ MP3 tracks to attempt" "$log" | grep -oE "^[0-9]+" | tail -1)
          mp3_now=$(beet ls 'format:mp3' 2>/dev/null | grep -c '^')
          snip="${attempted:-?} attempted · ${mp3_now} MP3 now" ;;
      esac
    fi
    # Age/snippet describe the last SUCCESS; if a newer attempt failed or was
    # lock-skipped, say so rather than hiding it behind the healthy line.
    case "$latest_status" in
      failed)       snip="${snip:+$snip, }latest attempt failed" ;;
      skipped_lock) snip="${snip:+$snip, }latest attempt skipped, lock busy" ;;
    esac
    detail="$age_h"
    [[ -n "$snip" ]] && detail="$age_h · $snip"
    if (( age_s > MAINT_LIMIT[$task] )) || [[ "$latest_status" == "failed" ]]; then
      mark_warn "$task" "$detail"
    else
      mark_ok "$task" "$detail"
    fi
  done

  # 4. Auth & state
  section "Auth & state"

  # Bandcamp cookie
  if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt ]]; then
    exp=$(awk -F'\t' '$6=="identity" {print $5; exit}' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)
    if [[ -n "${exp:-}" && "$exp" =~ ^[0-9]+$ && "$exp" -gt 0 ]]; then
      days=$(( (exp - NOW_TS) / 86400 ))
      if (( days < 14 )); then
        mark_warn "Bandcamp cookie" "$days days left — re-export soon"
      else
        mark_ok "Bandcamp cookie" "$days days left"
      fi
    else
      mark_warn "Bandcamp cookie" "could not parse expiry"
    fi
  else
    mark_warn "Bandcamp cookie" "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)"
  fi

  # Qobuz Web Player token (buy-link enrich cascade #2). The token is opaque
  # (no decodable expiry), so probe it live: a cheap authed search returns 200
  # while valid, 401 once it's logged out — at which point enrich-buy-url.py
  # silently drops Qobuz and falls through to iTunes.
  if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token ]]; then
    q_token=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)
    # Guard the app_id read with a file-exists check: `< missing 2>/dev/null`
    # still leaks the shell's own "No such file" redirection error (the redirect
    # is opened before 2>/dev/null takes effect). Most setups have a token but
    # no separate app_id, so this fired on every run.
    q_appid=798273057
    q_appid_file="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/app_id"
    [[ -f "$q_appid_file" ]] && q_appid=$(tr -d '\r\n' < "$q_appid_file")
    q_code=$(curl -s -o /dev/null --connect-timeout 5 --max-time 12 -A "Mozilla/5.0" \
      -H "X-App-Id: $q_appid" -H "X-User-Auth-Token: $q_token" -w '%{http_code}' \
      "https://www.qobuz.com/api.json/0.2/track/search?query=test&limit=1&app_id=$q_appid" 2>/dev/null)
    case "$q_code" in
      200) mark_ok   "Qobuz token" "valid (buy-link lookup live)" ;;
      401) mark_warn "Qobuz token" "EXPIRED — re-export X-User-Auth-Token to ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token" ;;
      *)   mark_warn "Qobuz token" "check failed (HTTP ${q_code:-none})" ;;
    esac
  else
    mark_warn "Qobuz token" "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)"
  fi

  # VPN egress
  # alembic itself rides gluetun's network namespace (network_mode:
  # service:gluetun), so its own outbound traffic IS gluetun's traffic —
  # checking alembic's own egress IP directly is a more direct signal than
  # asking Docker to introspect gluetun's health, and needs no Docker socket.
  egress_ip=$(curl -s --connect-timeout 5 --max-time 10 https://ifconfig.me 2>/dev/null)
  if [[ -n "$egress_ip" ]]; then
    mark_ok "VPN egress" "$egress_ip"
  else
    mark_warn "VPN egress" "could not determine egress IP — VPN may be down"
  fi

  # 5. Storage
  section "Storage"
  while read -r mp pct used total; do
    pct_n=${pct%\%}
    if (( pct_n > 90 )); then
      mark_warn "$mp" "$pct used ($used of $total)"
    else
      mark_ok "$mp" "$pct used ($used of $total)"
    fi
  done < <(df -h ${MUSIC_DATA_DIR:-/data/music} / 2>/dev/null | awk 'NR>1 && $1!~/tmpfs/ {print $6, $5, $3, $2}')

  # 6. Navidrome
  section "Navidrome"
  # Password via stdin (p@-), not argv, so it isn't exposed in ps/proc.
  ND_RESP=$(printf '%s' "$ND_PASS" | curl -s --connect-timeout 5 -G "$ND_BASE/rest/getScanStatus.view" \
    --data-urlencode "u=$ND_USER" \
    --data-urlencode "p@-" \
    --data-urlencode 'v=1.16.0' --data-urlencode 'c=status' --data-urlencode 'f=json' 2>/dev/null)
  ND_LINE=$(echo "$ND_RESP" | python3 -c "
import sys, json
try:
    d = json.load(sys.stdin)['subsonic-response']
    if d.get('status') == 'ok':
        s = d.get('scanStatus', {})
        cnt = s.get('count', '?')
        cnt_str = f'{cnt:,}' if isinstance(cnt, int) else str(cnt)
        print(f\"OK|last scan {s.get('lastScan','?')[:19]} · {cnt_str} items\")
    else:
        print(f\"WARN|status={d.get('status')}\")
except Exception as e:
    print(f\"WARN|unreachable: {e}\")
" 2>/dev/null)
  state=${ND_LINE%%|*}; detail=${ND_LINE##*|}
  if [[ "$state" == "OK" ]]; then mark_ok "Navidrome" "$detail"; else mark_warn "Navidrome" "$detail"; fi

  # 7. Library by format (info-only, no status dot)
  section "Library by format"
  beet ls -f '$format' 2>/dev/null | sort | uniq -c | sort -rn \
    | awk '{printf "   • %s — %d tracks\n", $2, $1}' \
    | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g'

} > "$OUT"

# ---- Build header banner and prepend ----
# Bold brand mark (⚗️ is literally the "ALEMBIC" unicode glyph — no generic
# music-note stand-in needed), a hostname chip in <code>, and an italic tally
# that reuses the same 🟢/🟠/⚪ dots as the body. When something needs
# attention, a <blockquote> callout mirrors the web app's .notice.warning
# panel; an all-clear run gets the calm .notice.success equivalent instead.
HOSTNAME_SHORT=$(hostname -s)
DATE_HUMAN=$(TZ="${TZ:-UTC}" date '+%a %Y-%m-%d %H:%M %Z')
HEADER="⚗️ <b>Alembic</b> · music pipeline · <code>${HOSTNAME_SHORT}</code> · ${DATE_HUMAN}
<i>🟢 ${OK} OK   🟠 ${WARN} warn   ⚪ ${SKIP} skip</i>"
if (( WARN > 0 )); then
  HEADER="${HEADER}
<blockquote>🟠 <b>${WARN} issue(s)</b> need attention — see below</blockquote>"
else
  HEADER="${HEADER}
<blockquote>🟢 All clear — nothing needs attention.</blockquote>"
fi

# Swap the __HEADER_PLACEHOLDER__ line for the (2-3 line) banner above. A
# straight prepend, not sed 1c — the banner's line count varies with the
# blockquote, which sed's `c` command can't take as a variable easily.
tail -n +2 "$OUT" > "${OUT}.body"
{ printf '%s\n' "$HEADER"; cat "${OUT}.body"; } > "$OUT"
rm -f "${OUT}.body"

# Always mirror the one-line summary to syslog so journalctl shows last-run
# state even if Telegram is broken. Warnings get a separate WARN tag.
slog "status: $OK ok, $WARN warn, $SKIP skip (see $OUT)"
if (( WARN > 0 )); then
  slog "WARN: pipeline-status flagged $WARN issue(s) — see $OUT"
fi

# Send to Telegram with --html: the digest is Telegram-HTML (see header
# comment), so notify-telegram.sh must send it with parse_mode=HTML. If the
# send fails, log to syslog so the absence of a message in the chat has a
# corresponding journal entry to grep for.
if [[ -x ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh ]]; then
  if ! ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh --html < "$OUT" 2>/tmp/tg-err; then
    slog "WARN: Telegram send failed: $(cat /tmp/tg-err 2>/dev/null | head -c 200)"
    rm -f /tmp/tg-err
  fi
fi