andrew
55a059b6da
sync-bandcamp.sh set a hardcoded PATH at startup that left out /opt/venv/bin, which is where beet lives in this image. Every time the Bandcamp sync had a new purchase to import, its call into import-track.sh would run beet import with that broken PATH, fail with "command not found", and import-track.sh would just log the exit code and carry on, so sync-bandcamp.sh still reported success. The purchase sat on disk, never entered the beets library, and never showed up in Navidrome. This went unnoticed for weeks because most daily syncs have nothing new to import, so the broken code path rarely ran. Fixed the same copy-pasted PATH line in upgrade-mp3-to-flac.sh (which also calls beet directly) and notify-telegram.sh (harmless there, but fixed for consistency). Also moved the post-import duplicate cleanup (replace-with-better.sh and dedup-library.sh) out of sync-bandcamp.sh and into import-track.sh itself, so every import gets the same cleanup, not just Bandcamp purchases. A manual import or SMB drop that happens to match something already in the library no longer leaves a duplicate copy sitting there until someone runs the dedup review by hand.
191 lines
8.2 KiB
Bash
Executable File
191 lines
8.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# upgrade-mp3-to-flac.sh — for every MP3 currently in the library, ask sldl
|
|
# to find a FLAC version of the same track. If found, replace the MP3.
|
|
# If not found, the MP3 stays (we don't lose anything).
|
|
#
|
|
# Strategy:
|
|
# 1. Build a CSV from beets with one row per MP3 track.
|
|
# 2. Run sldl with --format flac (HARD format requirement, not preferred —
|
|
# sldl will skip any track where no FLAC is available rather than
|
|
# falling back to MP3).
|
|
# 3. For each FLAC that arrives in the staging dropbox, run
|
|
# replace-with-better.sh which already knows the FLAC > MP3 rule.
|
|
#
|
|
# Conservative — if Soulseek peers are slow that day or the FLAC isn't
|
|
# available, we leave the MP3 alone. Re-runs are cheap; sldl's m3u skip
|
|
# logic means it won't re-search tracks it has already FLAC-found.
|
|
#
|
|
# Usage:
|
|
# upgrade-mp3-to-flac.sh # run the scanner end-to-end
|
|
# upgrade-mp3-to-flac.sh --csv-only # just write the CSV; don't run sldl
|
|
|
|
set -euo pipefail
|
|
# /opt/venv/bin must lead PATH -- `beet` (used heavily below) lives there.
|
|
PATH=/opt/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
export PATH
|
|
|
|
# These helpers are the same ones import-track.sh sources and runs under its
|
|
# own set -euo pipefail on every manual import, so they are already strict-safe.
|
|
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
|
source "${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh"
|
|
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
|
source "${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh"
|
|
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
|
source "${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh"
|
|
|
|
CSV_ONLY=0
|
|
[[ "${1:-}" == "--csv-only" ]] && CSV_ONLY=1
|
|
|
|
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/upgrade-mp3-$(date +%Y%m%d-%H%M%S).log
|
|
STAGING_HOST=${MUSIC_DATA_DIR:-/data/music}/sldl-dropbox/_upgrade
|
|
QUARANTINE_DIR=${MUSIC_DATA_DIR:-/data/music}/Songs/untagged
|
|
CSV_HOST=${ALEMBIC_CONFIG_DIR:-/config}/pipeline/_upgrade.csv
|
|
SLDL_BIN=${PIPELINE_DIR:-/app/pipeline}/sldl
|
|
|
|
mkdir -p "$(dirname "$LOG")" "$STAGING_HOST"
|
|
log() { echo "[$(date -Iseconds)] $*" | tee -a "$LOG"; }
|
|
|
|
log "=== MP3 upgrade scan starting ==="
|
|
|
|
# Build CSV from beets. sldl's CSV format: comma-separated with header
|
|
# Artist, Title, Album, Length (sec), URI (optional). We omit URI; sldl
|
|
# falls back to a Soulseek text search when no URI is given.
|
|
log "Querying beets for MP3 tracks..."
|
|
beet ls -f '$artist|$title|$album|$length' 'format:mp3' 2>/dev/null > /tmp/mp3-rows.txt
|
|
COUNT=$(wc -l < /tmp/mp3-rows.txt)
|
|
log " $COUNT MP3 tracks to attempt"
|
|
|
|
# Convert | to , and quote fields that contain commas
|
|
{
|
|
echo "Artist,Title,Album,Length"
|
|
awk -F'|' 'BEGIN{OFS=""} {
|
|
for (i=1; i<=NF; i++) {
|
|
g = $i; gsub(/"/, "\"\"", g)
|
|
if (g ~ /[,"]/) g = "\"" g "\""
|
|
$i = g
|
|
}
|
|
print $1 "," $2 "," $3 "," $4
|
|
}' /tmp/mp3-rows.txt
|
|
} > "$CSV_HOST"
|
|
|
|
log " CSV written: $CSV_HOST ($(wc -l < "$CSV_HOST") lines incl. header)"
|
|
|
|
if [[ $CSV_ONLY -eq 1 ]]; then
|
|
log "--csv-only flag — exiting before sldl run"
|
|
exit 0
|
|
fi
|
|
|
|
# Pull Soulseek credentials from any rendered playlist conf (they all carry the
|
|
# same user/pass). Don't hardcode a specific playlist name: pick the first
|
|
# non-underscore .conf that exists. Guarded so a missing conf yields empty
|
|
# creds (sldl then fails and is handled) rather than aborting under set -e.
|
|
_CRED_CONF=$(ls "${ALEMBIC_CONFIG_DIR:-/config}"/pipeline/*.conf 2>/dev/null | grep -v '/_' | head -1 || true)
|
|
SOULSEEK_USER=$(sed -n 's/^user *= *//p' "${_CRED_CONF:-/nonexistent}" 2>/dev/null | tr -d ' ' || true)
|
|
SOULSEEK_PASS=$(sed -n 's/^pass *= *//p' "${_CRED_CONF:-/nonexistent}" 2>/dev/null | tr -d ' ' || true)
|
|
|
|
log "Running sldl with --format flac (HARD requirement, no MP3/WAV fallback)"
|
|
# sldl routinely exits non-zero (skipped/partial searches), which is normal
|
|
# here. Capture the code without letting set -e abort: init to 0, override in
|
|
# the || branch. (`cmd; RC=$?` would abort at cmd before RC=$? ran.)
|
|
SLDL_EXIT=0
|
|
"$SLDL_BIN" \
|
|
"$CSV_HOST" \
|
|
--input-type csv \
|
|
--user "$SOULSEEK_USER" \
|
|
--pass "$SOULSEEK_PASS" \
|
|
--path "$STAGING_HOST" \
|
|
--format flac \
|
|
--pref-min-bitrate 320 \
|
|
--concurrent-downloads 8 \
|
|
--search-timeout 30000 \
|
|
--strict-title \
|
|
--strict-artist \
|
|
--no-browse-folder \
|
|
--no-skip-existing \
|
|
--verbose \
|
|
>> "$LOG" 2>&1 || SLDL_EXIT=$?
|
|
log "sldl exit code: $SLDL_EXIT"
|
|
|
|
# Guard: this script is FLAC-upgrade-only. If sldl ever drops anything else
|
|
# in staging (WAV/MP3/etc.), purge it so the import-leftover fallback in
|
|
# replace-with-better.sh doesn't pull non-FLACs into the library.
|
|
NON_FLAC=$(find "$STAGING_HOST" -type f ! -iname "*.flac" ! -name "*.incomplete" 2>/dev/null)
|
|
if [[ -n "$NON_FLAC" ]]; then
|
|
log "Purging non-FLAC files from staging:"
|
|
echo "$NON_FLAC" | tee -a "$LOG"
|
|
echo "$NON_FLAC" | xargs -r rm -f || true
|
|
fi
|
|
|
|
NEW_FLAC_COUNT=$(find "$STAGING_HOST" -type f -iname "*.flac" 2>/dev/null | wc -l)
|
|
log "Staging has $NEW_FLAC_COUNT FLAC file(s) ready for upgrade"
|
|
|
|
if [[ "$NEW_FLAC_COUNT" -eq 0 ]]; then
|
|
log "Nothing to upgrade. Done."
|
|
exit 0
|
|
fi
|
|
|
|
# === Treat staging files like new playlist downloads ===
|
|
# Same hygiene + prep as run-playlist.sh, applied to /sldl-dropbox/_upgrade
|
|
# before any of these files enter the library.
|
|
|
|
# Quarantine FLACs with missing ARTIST/TITLE — same rule as playlist runs.
|
|
# Prevents the __.wav-style "untagged files in /Library/ root" failure mode.
|
|
QUARANTINED=$(quarantine_untagged "$STAGING_HOST" "$QUARANTINE_DIR" "$LOG" || echo 0)
|
|
[[ "$QUARANTINED" -gt 0 ]] && log "Quarantined $QUARANTINED file(s) to $QUARANTINE_DIR"
|
|
|
|
# Recount post-quarantine
|
|
NEW_FLAC_COUNT=$(find "$STAGING_HOST" -type f -iname "*.flac" 2>/dev/null | wc -l)
|
|
[[ "$NEW_FLAC_COUNT" -eq 0 ]] && { log "Nothing left after quarantine. Done."; exit 0; }
|
|
|
|
# Strip MB IDs (prevents Navidrome album fragmentation). Scoped to staging:
|
|
# the library-wide strip-mb-tags.sh runs weekly anyway and wouldn't touch
|
|
# staging FLACs (they aren't in /Library/ yet).
|
|
MB_STRIPPED=$(strip_mb_tags "$STAGING_HOST" "$LOG" || echo 0)
|
|
log "Stripped MB tags from $MB_STRIPPED file(s) in staging"
|
|
|
|
# ALBUMARTIST fallback — safety net for no-match leftovers whose uploader
|
|
# didn't set ALBUMARTIST. For matched files this is overwritten by the
|
|
# tag-copy step inside replace-with-better.sh (canonical MP3 tags win).
|
|
AA_SET=$(set_albumartist_fallback "$STAGING_HOST" "$LOG" || echo 0)
|
|
[[ "$AA_SET" -gt 0 ]] && log "Set ALBUMARTIST fallback on $AA_SET file(s)"
|
|
|
|
# ReplayGain + autocue tagging — same as run-playlist.sh.
|
|
log "Pre-tagging with ReplayGain + autocue"
|
|
PREPPED=$(prep_audio "$STAGING_HOST" "$LOG" || echo 0)
|
|
log "Pre-tagged $PREPPED file(s)"
|
|
|
|
# Use replace-with-better.sh against the staging dir. --copy-tags-from-existing
|
|
# makes it copy canonical tags (incl. GROUPING) from each matched MP3 onto
|
|
# the replacing FLAC before the swap, so playlist M3Us still resolve and the
|
|
# FLAC inherits spotify-retag's canonical metadata.
|
|
log "Running replace-with-better.sh on staging (--copy-tags-from-existing)"
|
|
RBE_EXIT=0
|
|
${PIPELINE_DIR:-/app/pipeline}/lib/replace-with-better.sh --apply --copy-tags-from-existing "$STAGING_HOST" >> "$LOG" 2>&1 || RBE_EXIT=$?
|
|
log "replace-with-better exit: $RBE_EXIT"
|
|
|
|
# Regenerate playlist M3Us. Upgraded tracks have new paths (FLAC vs MP3) but
|
|
# their GROUPING tags were carried over via --copy-tags-from-existing, so the
|
|
# beet-driven regen below resolves them to the new FLAC paths.
|
|
log "Regenerating playlist M3Us"
|
|
PLAYLISTS_DIR=${MUSIC_DATA_DIR:-/data/music}/playlists
|
|
mkdir -p "$PLAYLISTS_DIR"
|
|
M3U_COUNT=0
|
|
while IFS= read -r playlist; do
|
|
[[ -z "$playlist" ]] && continue
|
|
M3U_OUT="${PLAYLISTS_DIR}/${playlist}.m3u8"
|
|
BEET_OUTPUT=$(beet ls -f '$path' "grouping:${playlist}" 2>>"$LOG" || true)
|
|
TRACK_COUNT=$(echo -n "$BEET_OUTPUT" | grep -c '^' || true)
|
|
[[ "$TRACK_COUNT" -gt 0 ]] || continue
|
|
{ echo "#EXTM3U"; echo "$BEET_OUTPUT"; } > "$M3U_OUT"
|
|
log " $playlist.m3u8 — $TRACK_COUNT tracks"
|
|
M3U_COUNT=$((M3U_COUNT + 1))
|
|
done < <(beet ls -f '$grouping' 2>/dev/null | sort -u | grep -v '^$')
|
|
log "Regenerated $M3U_COUNT M3U file(s)"
|
|
|
|
# Cleanup staging
|
|
log "Cleaning up empty staging dir"
|
|
find "$STAGING_HOST" -type f -delete 2>>"$LOG" || true
|
|
find "$STAGING_HOST" -mindepth 1 -type d -empty -delete 2>>"$LOG" || true
|
|
|
|
log "=== MP3 upgrade scan done ==="
|