Stage 0: migrate pipeline scripts from /opt/sldl, scaffold repo
Moves all ~30 pipeline scripts, configs, and the vendored sldl binary into this repo (source /opt/sldl left untouched). Removes all docker exec/docker compose dependencies now that beets and sldl run in-process/as a subprocess of this container instead of via soulbeet/on-demand sldl containers. Replaces hardcoded host paths, Navidrome credentials, and Spotify credential sourcing with env-var-driven paths and shared credential loaders. Adds Dockerfile, entrypoint.sh, requirements.txt, docker-compose.snippet.yml, and the initial app DB schema.
This commit is contained in:
andrew
Executable
+180
@@ -0,0 +1,180 @@
|
||||
#!/bin/bash
|
||||
# upgrade-mp3-to-flac.sh — for every MP3 currently in the library, ask sldl
|
||||
# to find a FLAC version of the same track. If found, replace the MP3.
|
||||
# If not found, the MP3 stays (we don't lose anything).
|
||||
#
|
||||
# Strategy:
|
||||
# 1. Build a CSV from beets with one row per MP3 track.
|
||||
# 2. Run sldl with --format flac (HARD format requirement, not preferred —
|
||||
# sldl will skip any track where no FLAC is available rather than
|
||||
# falling back to MP3).
|
||||
# 3. For each FLAC that arrives in the staging dropbox, run
|
||||
# replace-with-better.sh which already knows the FLAC > MP3 rule.
|
||||
#
|
||||
# Conservative — if Soulseek peers are slow that day or the FLAC isn't
|
||||
# available, we leave the MP3 alone. Re-runs are cheap; sldl's m3u skip
|
||||
# logic means it won't re-search tracks it has already FLAC-found.
|
||||
#
|
||||
# Usage:
|
||||
# upgrade-mp3-to-flac.sh # run the scanner end-to-end
|
||||
# upgrade-mp3-to-flac.sh --csv-only # just write the CSV; don't run sldl
|
||||
|
||||
set -u
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
export PATH
|
||||
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
|
||||
CSV_ONLY=0
|
||||
[[ "${1:-}" == "--csv-only" ]] && CSV_ONLY=1
|
||||
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/upgrade-mp3-$(date +%Y%m%d-%H%M%S).log
|
||||
STAGING_HOST=${MUSIC_DATA_DIR:-/data/music}/sldl-dropbox/_upgrade
|
||||
QUARANTINE_DIR=${MUSIC_DATA_DIR:-/data/music}/Songs/untagged
|
||||
CSV_HOST=${ALEMBIC_CONFIG_DIR:-/config}/pipeline/_upgrade.csv
|
||||
SLDL_BIN=${PIPELINE_DIR:-/app/pipeline}/sldl
|
||||
|
||||
mkdir -p "$(dirname "$LOG")" "$STAGING_HOST"
|
||||
log() { echo "[$(date -Iseconds)] $*" | tee -a "$LOG"; }
|
||||
|
||||
log "=== MP3 upgrade scan starting ==="
|
||||
|
||||
# Build CSV from beets. sldl's CSV format: comma-separated with header
|
||||
# Artist, Title, Album, Length (sec), URI (optional). We omit URI; sldl
|
||||
# falls back to a Soulseek text search when no URI is given.
|
||||
log "Querying beets for MP3 tracks..."
|
||||
beet ls -f '$artist|$title|$album|$length' 'format:mp3' 2>/dev/null > /tmp/mp3-rows.txt
|
||||
COUNT=$(wc -l < /tmp/mp3-rows.txt)
|
||||
log " $COUNT MP3 tracks to attempt"
|
||||
|
||||
# Convert | to , and quote fields that contain commas
|
||||
{
|
||||
echo "Artist,Title,Album,Length"
|
||||
awk -F'|' 'BEGIN{OFS=""} {
|
||||
for (i=1; i<=NF; i++) {
|
||||
g = $i; gsub(/"/, "\"\"", g)
|
||||
if (g ~ /[,"]/) g = "\"" g "\""
|
||||
$i = g
|
||||
}
|
||||
print $1 "," $2 "," $3 "," $4
|
||||
}' /tmp/mp3-rows.txt
|
||||
} > "$CSV_HOST"
|
||||
|
||||
log " CSV written: $CSV_HOST ($(wc -l < "$CSV_HOST") lines incl. header)"
|
||||
|
||||
if [[ $CSV_ONLY -eq 1 ]]; then
|
||||
log "--csv-only flag — exiting before sldl run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Pull credentials from any playlist conf (they're all the same)
|
||||
SOULSEEK_USER=$(sed -n 's/^user *= *//p' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/y2k.conf | tr -d ' ')
|
||||
SOULSEEK_PASS=$(sed -n 's/^pass *= *//p' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/y2k.conf | tr -d ' ')
|
||||
|
||||
log "Running sldl with --format flac (HARD requirement, no MP3/WAV fallback)"
|
||||
"$SLDL_BIN" \
|
||||
"$CSV_HOST" \
|
||||
--input-type csv \
|
||||
--user "$SOULSEEK_USER" \
|
||||
--pass "$SOULSEEK_PASS" \
|
||||
--path "$STAGING_HOST" \
|
||||
--format flac \
|
||||
--pref-min-bitrate 320 \
|
||||
--concurrent-downloads 8 \
|
||||
--search-timeout 30000 \
|
||||
--strict-title \
|
||||
--strict-artist \
|
||||
--no-browse-folder \
|
||||
--no-skip-existing \
|
||||
--verbose \
|
||||
>> "$LOG" 2>&1
|
||||
SLDL_EXIT=$?
|
||||
log "sldl exit code: $SLDL_EXIT"
|
||||
|
||||
# Guard: this script is FLAC-upgrade-only. If sldl ever drops anything else
|
||||
# in staging (WAV/MP3/etc.), purge it so the import-leftover fallback in
|
||||
# replace-with-better.sh doesn't pull non-FLACs into the library.
|
||||
NON_FLAC=$(find "$STAGING_HOST" -type f ! -iname "*.flac" ! -name "*.incomplete" 2>/dev/null)
|
||||
if [[ -n "$NON_FLAC" ]]; then
|
||||
log "Purging non-FLAC files from staging:"
|
||||
echo "$NON_FLAC" | tee -a "$LOG"
|
||||
echo "$NON_FLAC" | xargs -r rm -f
|
||||
fi
|
||||
|
||||
NEW_FLAC_COUNT=$(find "$STAGING_HOST" -type f -iname "*.flac" 2>/dev/null | wc -l)
|
||||
log "Staging has $NEW_FLAC_COUNT FLAC file(s) ready for upgrade"
|
||||
|
||||
if [[ "$NEW_FLAC_COUNT" -eq 0 ]]; then
|
||||
log "Nothing to upgrade. Done."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# === Treat staging files like new playlist downloads ===
|
||||
# Same hygiene + prep as run-playlist.sh, applied to /sldl-dropbox/_upgrade
|
||||
# before any of these files enter the library.
|
||||
|
||||
# Quarantine FLACs with missing ARTIST/TITLE — same rule as playlist runs.
|
||||
# Prevents the __.wav-style "untagged files in /Library/ root" failure mode.
|
||||
QUARANTINED=$(quarantine_untagged "$STAGING_HOST" "$QUARANTINE_DIR" "$LOG" || echo 0)
|
||||
[[ "$QUARANTINED" -gt 0 ]] && log "Quarantined $QUARANTINED file(s) to $QUARANTINE_DIR"
|
||||
|
||||
# Recount post-quarantine
|
||||
NEW_FLAC_COUNT=$(find "$STAGING_HOST" -type f -iname "*.flac" 2>/dev/null | wc -l)
|
||||
[[ "$NEW_FLAC_COUNT" -eq 0 ]] && { log "Nothing left after quarantine. Done."; exit 0; }
|
||||
|
||||
# Strip MB IDs (prevents Navidrome album fragmentation). Scoped to staging:
|
||||
# the library-wide strip-mb-tags.sh runs weekly anyway and wouldn't touch
|
||||
# staging FLACs (they aren't in /Library/ yet).
|
||||
MB_STRIPPED=$(strip_mb_tags "$STAGING_HOST" "$LOG" || echo 0)
|
||||
log "Stripped MB tags from $MB_STRIPPED file(s) in staging"
|
||||
|
||||
# ALBUMARTIST fallback — safety net for no-match leftovers whose uploader
|
||||
# didn't set ALBUMARTIST. For matched files this is overwritten by the
|
||||
# tag-copy step inside replace-with-better.sh (canonical MP3 tags win).
|
||||
AA_SET=$(set_albumartist_fallback "$STAGING_HOST" "$LOG" || echo 0)
|
||||
[[ "$AA_SET" -gt 0 ]] && log "Set ALBUMARTIST fallback on $AA_SET file(s)"
|
||||
|
||||
# ReplayGain + autocue tagging — same as run-playlist.sh.
|
||||
log "Pre-tagging with ReplayGain + autocue"
|
||||
PREPPED=$(prep_audio "$STAGING_HOST" "$LOG" || echo 0)
|
||||
log "Pre-tagged $PREPPED file(s)"
|
||||
|
||||
# Use replace-with-better.sh against the staging dir. --copy-tags-from-existing
|
||||
# makes it copy canonical tags (incl. GROUPING) from each matched MP3 onto
|
||||
# the replacing FLAC before the swap, so playlist M3Us still resolve and the
|
||||
# FLAC inherits spotify-retag's canonical metadata.
|
||||
log "Running replace-with-better.sh on staging (--copy-tags-from-existing)"
|
||||
${PIPELINE_DIR:-/app/pipeline}/lib/replace-with-better.sh --apply --copy-tags-from-existing "$STAGING_HOST" >> "$LOG" 2>&1
|
||||
RBE_EXIT=$?
|
||||
log "replace-with-better exit: $RBE_EXIT"
|
||||
|
||||
# Regenerate playlist M3Us. Upgraded tracks have new paths (FLAC vs MP3) but
|
||||
# their GROUPING tags were carried over via --copy-tags-from-existing, so the
|
||||
# beet-driven regen below resolves them to the new FLAC paths.
|
||||
log "Regenerating playlist M3Us"
|
||||
PLAYLISTS_DIR=${MUSIC_DATA_DIR:-/data/music}/playlists
|
||||
mkdir -p "$PLAYLISTS_DIR"
|
||||
M3U_COUNT=0
|
||||
while IFS= read -r playlist; do
|
||||
[[ -z "$playlist" ]] && continue
|
||||
M3U_OUT="${PLAYLISTS_DIR}/${playlist}.m3u8"
|
||||
BEET_OUTPUT=$(beet ls -f '$path' "grouping:${playlist}" 2>>"$LOG" || true)
|
||||
TRACK_COUNT=$(echo -n "$BEET_OUTPUT" | grep -c '^')
|
||||
[[ "$TRACK_COUNT" -gt 0 ]] || continue
|
||||
{ echo "#EXTM3U"; echo "$BEET_OUTPUT"; } > "$M3U_OUT"
|
||||
log " $playlist.m3u8 — $TRACK_COUNT tracks"
|
||||
M3U_COUNT=$((M3U_COUNT + 1))
|
||||
done < <(beet ls -f '$grouping' 2>/dev/null | sort -u | grep -v '^$')
|
||||
log "Regenerated $M3U_COUNT M3U file(s)"
|
||||
|
||||
# Cleanup staging
|
||||
log "Cleaning up empty staging dir"
|
||||
find "$STAGING_HOST" -type f -delete 2>>"$LOG" || true
|
||||
find "$STAGING_HOST" -mindepth 1 -type d -empty -delete 2>>"$LOG" || true
|
||||
|
||||
log "=== MP3 upgrade scan done ==="
|
||||
Reference in New Issue
Block a user