Stage 0: migrate pipeline scripts from /opt/sldl, scaffold repo
Moves all ~30 pipeline scripts, configs, and the vendored sldl binary into this repo (source /opt/sldl left untouched). Removes all docker exec/docker compose dependencies now that beets and sldl run in-process/as a subprocess of this container instead of via soulbeet/on-demand sldl containers. Replaces hardcoded host paths, Navidrome credentials, and Spotify credential sourcing with env-var-driven paths and shared credential loaders. Adds Dockerfile, entrypoint.sh, requirements.txt, docker-compose.snippet.yml, and the initial app DB schema.
This commit is contained in:
andrew
Executable
+189
@@ -0,0 +1,189 @@
|
||||
#!/bin/bash
|
||||
# run-playlist.sh
|
||||
#
|
||||
# Runs sldl (vendored binary, subprocess of this same container — which
|
||||
# rides gluetun's VPN network namespace) for a Spotify playlist, tags
|
||||
# downloaded files with the playlist name, imports via beets (also in-
|
||||
# process in this container), and regenerates a Navidrome-readable M3U
|
||||
# playlist.
|
||||
#
|
||||
# Usage: ./run-playlist.sh [--no-m3u] <playlist_name>
|
||||
# --no-m3u Download and import but skip M3U generation (e.g. liked songs)
|
||||
|
||||
set -euo pipefail
|
||||
# Commands that are *allowed* to exit non-zero must be wrapped explicitly,
|
||||
# either by `cmd && X=0 || X=$?` to capture exit code, or `cmd || true` when
|
||||
# we genuinely don't care. Everything else aborts the run on failure.
|
||||
|
||||
NO_M3U=false
|
||||
if [[ "${1:-}" == "--no-m3u" ]]; then
|
||||
NO_M3U=true
|
||||
shift
|
||||
fi
|
||||
|
||||
PLAYLIST_NAME="${1:?Usage: $0 [--no-m3u] <playlist_name>}"
|
||||
|
||||
# ==== Paths ====
|
||||
CONFIG_FILE=${ALEMBIC_CONFIG_DIR:-/config}/pipeline/${PLAYLIST_NAME}.conf
|
||||
DROPBOX=${MUSIC_DATA_DIR:-/data/music}/sldl-dropbox/${PLAYLIST_NAME}
|
||||
PLAYLISTS_DIR=${MUSIC_DATA_DIR:-/data/music}/playlists
|
||||
QUARANTINE_DIR=${MUSIC_DATA_DIR:-/data/music}/Songs/untagged
|
||||
LOG_DIR=${ALEMBIC_CONFIG_DIR:-/config}/logs
|
||||
LOG=${LOG_DIR}/${PLAYLIST_NAME}-$(date +%Y%m%d).log
|
||||
SLDL_BIN=${PIPELINE_DIR:-/app/pipeline}/sldl
|
||||
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
|
||||
# ==== Setup ====
|
||||
mkdir -p "$LOG_DIR" "$DROPBOX" "$PLAYLISTS_DIR"
|
||||
|
||||
log() {
|
||||
echo "[$(date -Iseconds)] $*" | tee -a "$LOG"
|
||||
}
|
||||
|
||||
log "=== Starting playlist run: $PLAYLIST_NAME ==="
|
||||
|
||||
# ==== Sanity checks ====
|
||||
if [[ ! -f "$CONFIG_FILE" ]]; then
|
||||
log "ERROR: Config file not found at $CONFIG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==== Run sldl (vendored binary, subprocess of this container) ====
|
||||
log "Running sldl for: $PLAYLIST_NAME"
|
||||
|
||||
# sldl exit code is informational: per-track Soulseek failures are normal and
|
||||
# do NOT mean we should abort — we still want to tag/import whatever it did
|
||||
# download. A non-zero exit here is logged but not fatal.
|
||||
#
|
||||
# Hard timeout: without it a stuck sldl hangs FOREVER holding the shared lock,
|
||||
# which silently skips every later-scheduled playlist for the night. (Seen
|
||||
# 2026-06-18: a transient Spotify client-creds failure made sldl fall back to
|
||||
# interactive OAuth — "manually open: https://accounts.spotify.com/..." — and
|
||||
# it waited 7h on a browser callback that never comes.) timeout TERMs the run
|
||||
# and KILLs after a grace period; there's no leftover container to clean up
|
||||
# now that sldl is a plain subprocess instead of a docker-compose service.
|
||||
SLDL_TIMEOUT="${SLDL_TIMEOUT:-2700}" # 45 min; override via env
|
||||
SLDL_EXIT=0
|
||||
timeout --kill-after=30s "$SLDL_TIMEOUT" \
|
||||
"$SLDL_BIN" -c "$CONFIG_FILE" >> "$LOG" 2>&1 \
|
||||
|| SLDL_EXIT=$?
|
||||
if [[ "$SLDL_EXIT" -eq 124 || "$SLDL_EXIT" -eq 137 ]]; then
|
||||
log "ERROR: sldl exceeded ${SLDL_TIMEOUT}s and was killed (likely a hang)"
|
||||
fi
|
||||
log "sldl finished with exit code $SLDL_EXIT"
|
||||
|
||||
# ==== Tag every file in this playlist's dropbox with GROUPING=<playlist> ====
|
||||
# We tag all files in the dropbox (not just files new to this run) for
|
||||
# two reasons:
|
||||
# 1. If a previous sldl run crashed mid-flight, the files it managed to
|
||||
# download are still sitting here and would otherwise be skipped.
|
||||
# 2. Re-tagging an already-correct file is a no-op (metaflac/id3v2 just
|
||||
# overwrite with the same value).
|
||||
log "Tagging files in $DROPBOX with GROUPING=$PLAYLIST_NAME"
|
||||
|
||||
NEW_FLAC_COUNT=0
|
||||
NEW_MP3_COUNT=0
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
metaflac --remove-tag=GROUPING --set-tag="GROUPING=${PLAYLIST_NAME}" "$file" 2>>"$LOG" || true
|
||||
NEW_FLAC_COUNT=$((NEW_FLAC_COUNT + 1))
|
||||
done < <(find "$DROPBOX" -name "*.flac" -type f -print0)
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
id3v2 --TIT1 "${PLAYLIST_NAME}" "$file" 2>>"$LOG" || true
|
||||
NEW_MP3_COUNT=$((NEW_MP3_COUNT + 1))
|
||||
done < <(find "$DROPBOX" -name "*.mp3" -type f -print0)
|
||||
|
||||
log "Tagged $NEW_FLAC_COUNT FLAC and $NEW_MP3_COUNT MP3 files"
|
||||
|
||||
# ==== Clean up any .incomplete files before beets touches them ====
|
||||
log "Cleaning up .incomplete files in dropbox"
|
||||
find "$DROPBOX" -name "*.incomplete" -type f -delete 2>>"$LOG" || true
|
||||
|
||||
# ==== Rewrite tags from Spotify (source of truth for matched tracks) ====
|
||||
# Reads Spotify creds + playlist URL from the same conf sldl used. For each file
|
||||
# in the dropbox that matches a Spotify playlist track, overwrites ARTIST
|
||||
# (semicolon-joined), ALBUMARTIST (primary), ALBUM, TITLE, TRACKNUMBER,
|
||||
# DISCNUMBER, DATE. GROUPING is preserved. Files that don't match are left for
|
||||
# the fallback step below.
|
||||
SPOTIFY_URL=$(sed -n 's/^input *= *//p' "$CONFIG_FILE" | tr -d ' ')
|
||||
SPOTIFY_CLIENT_ID=$(sed -n 's/^spotify-id *= *//p' "$CONFIG_FILE" | tr -d ' ')
|
||||
SPOTIFY_CLIENT_SECRET=$(sed -n 's/^spotify-secret *= *//p' "$CONFIG_FILE" | tr -d ' ')
|
||||
|
||||
if [[ -n "$SPOTIFY_URL" && -n "$SPOTIFY_CLIENT_ID" && -n "$SPOTIFY_CLIENT_SECRET" ]]; then
|
||||
log "Rewriting tags from Spotify for files in $DROPBOX"
|
||||
if SPOTIFY_CLIENT_ID="$SPOTIFY_CLIENT_ID" SPOTIFY_CLIENT_SECRET="$SPOTIFY_CLIENT_SECRET" \
|
||||
python3 ${PIPELINE_DIR:-/app/pipeline}/lib/spotify-retag.py "$SPOTIFY_URL" "$DROPBOX" >> "$LOG" 2>&1; then
|
||||
log "Spotify retag complete"
|
||||
else
|
||||
log "WARNING: Spotify retag failed (exit $?) — continuing with sldl-supplied tags"
|
||||
fi
|
||||
else
|
||||
log "Skipping Spotify retag — missing input/spotify-id/spotify-secret in $CONFIG_FILE"
|
||||
fi
|
||||
|
||||
# ==== Quarantine any files still missing essential tags ====
|
||||
QUARANTINED=$(quarantine_untagged "$DROPBOX" "$QUARANTINE_DIR" "$LOG" || echo 0)
|
||||
if [[ "$QUARANTINED" -gt 0 ]]; then
|
||||
log "Quarantined $QUARANTINED file(s) to $QUARANTINE_DIR — fix tags and re-import via import-track.sh"
|
||||
fi
|
||||
|
||||
# ==== Set ALBUMARTIST fallback so Navidrome doesn't coin ghost combined artists ====
|
||||
AA_SET=$(set_albumartist_fallback "$DROPBOX" "$LOG" || echo 0)
|
||||
[[ "$AA_SET" -gt 0 ]] && log "Set ALBUMARTIST fallback on $AA_SET file(s) (primary artist before first separator)"
|
||||
|
||||
# ==== Pre-tag with loudgain (ReplayGain) + autocue (liq_* cue/overlay tags) ====
|
||||
# Done here, on the dropbox copy, so Navidrome/AzuraCast see fully-prepped files
|
||||
# the moment beets moves them into /music.
|
||||
PREPPED=$(prep_audio "$DROPBOX" "$LOG" || echo 0)
|
||||
log "Pre-tagged $PREPPED file(s) with ReplayGain + autocue"
|
||||
|
||||
# ==== Trigger beets import (in-process, same container) ====
|
||||
log "Running beets import on $DROPBOX"
|
||||
# Beets sometimes returns non-zero when it skips duplicates — not fatal.
|
||||
BEETS_EXIT=0
|
||||
beet import -q -s "$DROPBOX" >> "$LOG" 2>&1 || BEETS_EXIT=$?
|
||||
log "beets import finished with exit code $BEETS_EXIT"
|
||||
|
||||
# ==== Diagnostic: verify tags exist in beets DB ====
|
||||
# || true, not || echo 0: under pipefail a failed `beet ls` would otherwise
|
||||
# append a second "0" after the one wc already printed ("0\n0").
|
||||
TAGGED_COUNT=$(beet ls "grouping:${PLAYLIST_NAME}" 2>>"$LOG" | wc -l || true)
|
||||
log "Beets has $TAGGED_COUNT tracks tagged with grouping:$PLAYLIST_NAME"
|
||||
|
||||
# ==== Regenerate M3U based on beets library state ====
|
||||
if [[ "$NO_M3U" == true ]]; then
|
||||
log "Skipping M3U generation (--no-m3u)"
|
||||
else
|
||||
M3U_OUT="${PLAYLISTS_DIR}/${PLAYLIST_NAME}.m3u8"
|
||||
log "Regenerating M3U at $M3U_OUT from beets library"
|
||||
|
||||
# Capture beet output to a variable first, then write to file.
|
||||
BEET_OUTPUT=$(beet ls -f '$path' "grouping:${PLAYLIST_NAME}" 2>>"$LOG" || true)
|
||||
# grep -c '^' exits 1 on empty input, which would trip set -e and kill the
|
||||
# whole run silently right here. Count lines a way that's safe on no-match.
|
||||
if [[ -z "$BEET_OUTPUT" ]]; then
|
||||
TRACK_COUNT=0
|
||||
else
|
||||
TRACK_COUNT=$(printf '%s\n' "$BEET_OUTPUT" | wc -l)
|
||||
fi
|
||||
|
||||
if [[ "$TRACK_COUNT" -gt 0 ]]; then
|
||||
{
|
||||
echo "#EXTM3U"
|
||||
echo "$BEET_OUTPUT"
|
||||
} > "$M3U_OUT"
|
||||
log "M3U updated with $TRACK_COUNT tracks"
|
||||
else
|
||||
log "WARNING: beet list returned no tracks for grouping:$PLAYLIST_NAME — not updating M3U"
|
||||
fi
|
||||
fi
|
||||
|
||||
log "=== Finished playlist run: $PLAYLIST_NAME ==="
|
||||
log ""
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user