andrew
d756aab7fc
Two bugs conspired to fill the library with .1.flac duplicates on the night of 2026-07-15/16: 1. sldl derives its skip-index folder from the input: the Spotify playlist display name before 0.6.2, the CSV filename after. The switch stranded every playlist's download history in the old emoji-named subfolder, so sldl started from an empty index and re-downloaded every playlist in full. Fix: pin index-path in _template.conf, seed the pinned file from all stranded indexes (merge-sldl-indexes.py, called from run-playlist.sh whenever the pinned index is missing or older than a stranded one). Recovered rows keep the CSV-era identity fields but are marked downloaded, so tracks that failed last night yet exist in the library since months ago are not fetched again. 2. The safety net that should have flagged the flood, dedup-library.sh, has silently reported 0 groups since the 2026-07-08 path rewrite: host_path() still assumed /music/... display paths and prepended the library dir to already-correct /data/music/Library/... paths, so every candidate failed the -f check. Fix: pass real paths through unchanged, keep the /music prefix swap only for legacy entries. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
228 lines
10 KiB
Bash
Executable File
228 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
# run-playlist.sh
|
|
#
|
|
# Runs sldl (vendored binary, subprocess of this same container — which
|
|
# rides gluetun's VPN network namespace) for a Spotify playlist, tags
|
|
# downloaded files with the playlist name, imports via beets (also in-
|
|
# process in this container), and regenerates a Navidrome-readable M3U
|
|
# playlist.
|
|
#
|
|
# Usage: ./run-playlist.sh [--no-m3u] <playlist_name>
|
|
# --no-m3u Download and import but skip M3U generation (e.g. liked songs)
|
|
|
|
set -euo pipefail
|
|
# Commands that are *allowed* to exit non-zero must be wrapped explicitly,
|
|
# either by `cmd && X=0 || X=$?` to capture exit code, or `cmd || true` when
|
|
# we genuinely don't care. Everything else aborts the run on failure.
|
|
|
|
NO_M3U=false
|
|
if [[ "${1:-}" == "--no-m3u" ]]; then
|
|
NO_M3U=true
|
|
shift
|
|
fi
|
|
|
|
PLAYLIST_NAME="${1:?Usage: $0 [--no-m3u] <playlist_name>}"
|
|
|
|
# ==== Paths ====
|
|
CONFIG_FILE=${ALEMBIC_CONFIG_DIR:-/config}/pipeline/${PLAYLIST_NAME}.conf
|
|
SPOTIFY_ENV=${ALEMBIC_CONFIG_DIR:-/config}/pipeline/_spotify.env
|
|
CSV_FILE=${ALEMBIC_CONFIG_DIR:-/config}/pipeline/${PLAYLIST_NAME}.csv
|
|
DROPBOX=${MUSIC_DATA_DIR:-/data/music}/sldl-dropbox/${PLAYLIST_NAME}
|
|
PLAYLISTS_DIR=${MUSIC_DATA_DIR:-/data/music}/playlists
|
|
QUARANTINE_DIR=${MUSIC_DATA_DIR:-/data/music}/Songs/untagged
|
|
LOG_DIR=${ALEMBIC_CONFIG_DIR:-/config}/logs
|
|
LOG=${LOG_DIR}/${PLAYLIST_NAME}-$(date +%Y%m%d).log
|
|
SLDL_BIN=${PIPELINE_DIR:-/app/pipeline}/sldl
|
|
|
|
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
|
source ${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
|
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
|
source ${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
|
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/m3u.sh
|
|
source ${PIPELINE_DIR:-/app/pipeline}/lib/m3u.sh
|
|
|
|
# ==== Setup ====
|
|
mkdir -p "$LOG_DIR" "$DROPBOX" "$PLAYLISTS_DIR"
|
|
|
|
log() {
|
|
echo "[$(date -Iseconds)] $*" | tee -a "$LOG"
|
|
}
|
|
|
|
log "=== Starting playlist run: $PLAYLIST_NAME ==="
|
|
|
|
# ==== Sanity checks ====
|
|
if [[ ! -f "$CONFIG_FILE" ]]; then
|
|
log "ERROR: Config file not found at $CONFIG_FILE"
|
|
exit 1
|
|
fi
|
|
|
|
# ==== Read the playlist from Spotify into a CSV sldl can search from ====
|
|
# sldl's own vendored Spotify client still calls GET /playlists/{id}/tracks,
|
|
# which Spotify removed in its February 2026 API changes. Grandfathered apps
|
|
# still get a pass there; apps created after that change get a hard 403
|
|
# regardless of auth method (client-credentials, or even a correctly-scoped,
|
|
# correctly-owned OAuth user token -- confirmed live, exit 134 unhandled
|
|
# APIException: Forbidden from Extractors.Spotify.GetPlaylist). sldl has no
|
|
# fallback to the still-working /items endpoint, so it can never read a
|
|
# playlist for a new app no matter what alembic hands it. Reading the
|
|
# playlist ourselves (via /items) and handing sldl a plain CSV instead
|
|
# sidesteps sldl's Spotify client entirely -- works the same for
|
|
# grandfathered and new apps alike.
|
|
SPOTIFY_URL=$(sed -n 's/^input *= *//p' "$CONFIG_FILE" | tr -d ' ')
|
|
[[ -f "$SPOTIFY_ENV" ]] && source "$SPOTIFY_ENV"
|
|
|
|
if [[ -z "$SPOTIFY_URL" || -z "${SPOTIFY_CLIENT_ID:-}" || -z "${SPOTIFY_CLIENT_SECRET:-}" ]]; then
|
|
log "ERROR: missing playlist URL in $CONFIG_FILE or Spotify credentials in $SPOTIFY_ENV"
|
|
exit 1
|
|
fi
|
|
|
|
log "Fetching playlist from Spotify: $SPOTIFY_URL"
|
|
if ! SPOTIFY_CLIENT_ID="$SPOTIFY_CLIENT_ID" SPOTIFY_CLIENT_SECRET="$SPOTIFY_CLIENT_SECRET" \
|
|
SPOTIFY_REFRESH_TOKEN="${SPOTIFY_REFRESH_TOKEN:-}" \
|
|
python3 "${PIPELINE_DIR:-/app/pipeline}/lib/spotify-playlist-csv.py" "$SPOTIFY_URL" "$CSV_FILE" >> "$LOG" 2>&1; then
|
|
log "ERROR: failed to fetch playlist from Spotify — see $LOG"
|
|
exit 1
|
|
fi
|
|
|
|
# Surface the fetched count in the timestamped summary. Zero tracks from a
|
|
# successful fetch is almost never a truly empty playlist -- it usually means
|
|
# Spotify hid the contents from the connected account (see the pointed errors
|
|
# in spotify-playlist-csv.py), so call it out instead of letting sldl no-op
|
|
# with a clean exit 0.
|
|
TRACK_COUNT=$(($(wc -l < "$CSV_FILE") - 1))
|
|
if [[ "$TRACK_COUNT" -le 0 ]]; then
|
|
log "WARNING: Spotify returned 0 visible tracks for this playlist -- nothing to download. If the playlist isn't actually empty, the connected Spotify account can't see its contents (it must own or collaborate on the playlist for newly created Spotify apps)."
|
|
else
|
|
log "Fetched $TRACK_COUNT track(s) from Spotify"
|
|
fi
|
|
|
|
# ==== Seed the pinned sldl skip index if it's missing or stranded ====
|
|
# The conf pins index-path to $DROPBOX/_index.csv (see _template.conf: sldl's
|
|
# default index location is derived from the input, so an input change strands
|
|
# the old index and the whole playlist re-downloads -- that's what produced
|
|
# the 2026-07-16 duplicate flood). If the pinned file is missing, or an index
|
|
# in one of sldl's old input-named subfolders is newer (i.e. sldl last wrote
|
|
# somewhere else), fold them all into the pinned location first.
|
|
INDEX_FILE="$DROPBOX/_index.csv"
|
|
if [[ ! -s "$INDEX_FILE" ]] || \
|
|
[[ -n "$(find "$DROPBOX" -mindepth 2 -maxdepth 2 -name _index.csv -newer "$INDEX_FILE" -print -quit 2>/dev/null)" ]]; then
|
|
log "Seeding pinned sldl index at $INDEX_FILE from prior indexes"
|
|
python3 "${PIPELINE_DIR:-/app/pipeline}/lib/merge-sldl-indexes.py" "$DROPBOX" "$INDEX_FILE" >> "$LOG" 2>&1 \
|
|
|| log "WARNING: index merge failed -- sldl may re-download tracks the library already has"
|
|
fi
|
|
|
|
# ==== Run sldl (vendored binary, subprocess of this container) ====
|
|
log "Running sldl for: $PLAYLIST_NAME"
|
|
|
|
# sldl exit code is informational: per-track Soulseek failures are normal and
|
|
# do NOT mean we should abort — we still want to tag/import whatever it did
|
|
# download. A non-zero exit here is logged but not fatal.
|
|
#
|
|
# Hard timeout: without it a stuck sldl hangs FOREVER holding the shared lock,
|
|
# which silently skips every later-scheduled playlist for the night. timeout
|
|
# TERMs the run and KILLs after a grace period; there's no leftover container
|
|
# to clean up now that sldl is a plain subprocess instead of a docker-compose
|
|
# service.
|
|
#
|
|
# -c "$CONFIG_FILE" still supplies Soulseek login, download path, quality
|
|
# settings, port, etc. -- the CSV positional arg + --input-type csv override
|
|
# just the input source (confirmed: sldl ignores the conf's own `input =`/
|
|
# `input-type =` lines when both are given).
|
|
SLDL_TIMEOUT="${SLDL_TIMEOUT:-2700}" # 45 min; override via env
|
|
SLDL_EXIT=0
|
|
timeout --kill-after=30s "$SLDL_TIMEOUT" \
|
|
"$SLDL_BIN" "$CSV_FILE" --input-type csv -c "$CONFIG_FILE" >> "$LOG" 2>&1 \
|
|
|| SLDL_EXIT=$?
|
|
if [[ "$SLDL_EXIT" -eq 124 || "$SLDL_EXIT" -eq 137 ]]; then
|
|
log "ERROR: sldl exceeded ${SLDL_TIMEOUT}s and was killed (likely a hang)"
|
|
elif [[ "$SLDL_EXIT" -eq 134 ]]; then
|
|
log "ERROR: sldl crashed (exit 134, unhandled exception) -- see the log above for the exception detail"
|
|
fi
|
|
log "sldl finished with exit code $SLDL_EXIT"
|
|
|
|
# ==== Tag every file in this playlist's dropbox with GROUPING=<playlist> ====
|
|
# We tag all files in the dropbox (not just files new to this run) for
|
|
# two reasons:
|
|
# 1. If a previous sldl run crashed mid-flight, the files it managed to
|
|
# download are still sitting here and would otherwise be skipped.
|
|
# 2. Re-tagging an already-correct file is a no-op (metaflac/id3v2 just
|
|
# overwrite with the same value).
|
|
log "Tagging files in $DROPBOX with GROUPING=$PLAYLIST_NAME"
|
|
|
|
NEW_FLAC_COUNT=0
|
|
NEW_MP3_COUNT=0
|
|
|
|
while IFS= read -r -d '' file; do
|
|
metaflac --remove-tag=GROUPING --set-tag="GROUPING=${PLAYLIST_NAME}" "$file" 2>>"$LOG" || true
|
|
NEW_FLAC_COUNT=$((NEW_FLAC_COUNT + 1))
|
|
done < <(find "$DROPBOX" -name "*.flac" -type f -print0)
|
|
|
|
while IFS= read -r -d '' file; do
|
|
id3v2 --TIT1 "${PLAYLIST_NAME}" "$file" 2>>"$LOG" || true
|
|
NEW_MP3_COUNT=$((NEW_MP3_COUNT + 1))
|
|
done < <(find "$DROPBOX" -name "*.mp3" -type f -print0)
|
|
|
|
log "Tagged $NEW_FLAC_COUNT FLAC and $NEW_MP3_COUNT MP3 files"
|
|
|
|
# ==== Clean up any .incomplete files before beets touches them ====
|
|
log "Cleaning up .incomplete files in dropbox"
|
|
find "$DROPBOX" -name "*.incomplete" -type f -delete 2>>"$LOG" || true
|
|
|
|
# ==== Rewrite tags from Spotify (source of truth for matched tracks) ====
|
|
# Uses the playlist URL and _spotify.env credentials already loaded for the
|
|
# CSV fetch above (confs no longer carry Spotify creds). For each file in the
|
|
# dropbox that matches a Spotify playlist track, overwrites ARTIST
|
|
# (semicolon-joined), ALBUMARTIST (primary), ALBUM, TITLE, TRACKNUMBER,
|
|
# DISCNUMBER, DATE. GROUPING is preserved. Files that don't match are left for
|
|
# the fallback step below.
|
|
log "Rewriting tags from Spotify for files in $DROPBOX"
|
|
if SPOTIFY_CLIENT_ID="$SPOTIFY_CLIENT_ID" SPOTIFY_CLIENT_SECRET="$SPOTIFY_CLIENT_SECRET" \
|
|
SPOTIFY_REFRESH_TOKEN="${SPOTIFY_REFRESH_TOKEN:-}" \
|
|
python3 ${PIPELINE_DIR:-/app/pipeline}/lib/spotify-retag.py "$SPOTIFY_URL" "$DROPBOX" >> "$LOG" 2>&1; then
|
|
log "Spotify retag complete"
|
|
else
|
|
log "WARNING: Spotify retag failed (exit $?) — continuing with sldl-supplied tags"
|
|
fi
|
|
|
|
# ==== Quarantine any files still missing essential tags ====
|
|
QUARANTINED=$(quarantine_untagged "$DROPBOX" "$QUARANTINE_DIR" "$LOG" || echo 0)
|
|
if [[ "$QUARANTINED" -gt 0 ]]; then
|
|
log "Quarantined $QUARANTINED file(s) to $QUARANTINE_DIR — fix tags and re-import via import-track.sh"
|
|
fi
|
|
|
|
# ==== Set ALBUMARTIST fallback so Navidrome doesn't coin ghost combined artists ====
|
|
AA_SET=$(set_albumartist_fallback "$DROPBOX" "$LOG" || echo 0)
|
|
[[ "$AA_SET" -gt 0 ]] && log "Set ALBUMARTIST fallback on $AA_SET file(s) (primary artist before first separator)"
|
|
|
|
# ==== Pre-tag with loudgain (ReplayGain) + autocue (liq_* cue/overlay tags) ====
|
|
# Done here, on the dropbox copy, so Navidrome/AzuraCast see fully-prepped files
|
|
# the moment beets moves them into /music.
|
|
PREPPED=$(prep_audio "$DROPBOX" "$LOG" || echo 0)
|
|
log "Pre-tagged $PREPPED file(s) with ReplayGain + autocue"
|
|
|
|
# ==== Trigger beets import (in-process, same container) ====
|
|
log "Running beets import on $DROPBOX"
|
|
# Beets sometimes returns non-zero when it skips duplicates — not fatal.
|
|
BEETS_EXIT=0
|
|
beet import -q -s "$DROPBOX" >> "$LOG" 2>&1 || BEETS_EXIT=$?
|
|
log "beets import finished with exit code $BEETS_EXIT"
|
|
|
|
# ==== Diagnostic: verify tags exist in beets DB ====
|
|
# || true, not || echo 0: under pipefail a failed `beet ls` would otherwise
|
|
# append a second "0" after the one wc already printed ("0\n0").
|
|
TAGGED_COUNT=$(beet ls "grouping:${PLAYLIST_NAME}" 2>>"$LOG" | wc -l || true)
|
|
log "Beets has $TAGGED_COUNT tracks tagged with grouping:$PLAYLIST_NAME"
|
|
|
|
# ==== Regenerate M3U based on beets library state ====
|
|
if [[ "$NO_M3U" == true ]]; then
|
|
log "Skipping M3U generation (--no-m3u)"
|
|
else
|
|
log "Regenerating M3U for $PLAYLIST_NAME from beets library"
|
|
regen_m3u "$PLAYLIST_NAME" "${PLAYLISTS_DIR}/${PLAYLIST_NAME}.m3u8"
|
|
fi
|
|
|
|
log "=== Finished playlist run: $PLAYLIST_NAME ==="
|
|
log ""
|
|
|
|
exit 0
|