Stage 0: migrate pipeline scripts from /opt/sldl, scaffold repo
Moves all ~30 pipeline scripts, configs, and the vendored sldl binary into this repo (source /opt/sldl left untouched). Removes all docker exec/docker compose dependencies now that beets and sldl run in-process/as a subprocess of this container instead of via soulbeet/on-demand sldl containers. Replaces hardcoded host paths, Navidrome credentials, and Spotify credential sourcing with env-var-driven paths and shared credential loaders. Adds Dockerfile, entrypoint.sh, requirements.txt, docker-compose.snippet.yml, and the initial app DB schema.
This commit is contained in:
andrew
Executable
+243
@@ -0,0 +1,243 @@
|
||||
#!/bin/bash
|
||||
# Walk a "stranded" download directory (files beets rejected as duplicates),
|
||||
# and for each file, compare against the existing library entry. If the new
|
||||
# file is higher quality (FLAC > MP3, then larger size), REPLACE the library
|
||||
# version. Otherwise delete the new file.
|
||||
#
|
||||
# Usage:
|
||||
# replace-with-better.sh # default scan: ${MUSIC_DATA_DIR:-/data/music}/downloads
|
||||
# replace-with-better.sh <scan_dir> # custom scan dir
|
||||
# replace-with-better.sh --apply # actually replace (default is dry-run)
|
||||
#
|
||||
# Run after sync-bandcamp.sh when the FLAC > MP3 dedup didn't pick up new
|
||||
# imports (beets quietly skipped them and left files in /downloads).
|
||||
|
||||
set -u
|
||||
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
|
||||
APPLY=0
|
||||
COPY_TAGS=0
|
||||
SCAN_DIR=${MUSIC_DATA_DIR:-/data/music}/downloads
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--apply) APPLY=1 ;;
|
||||
--copy-tags-from-existing) COPY_TAGS=1 ;;
|
||||
*) SCAN_DIR="$arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Copy canonical tags from an existing (already spotify-retagged) library track
|
||||
# onto a new FLAC, replacing whatever uploader-supplied tags it came with.
|
||||
# Uses beets as the source of truth — same fields beets/Navidrome care about.
|
||||
copy_tags_from_existing() {
|
||||
local existing_id="$1" # beets item id of the existing library track
|
||||
local new_file="$2" # host path of new FLAC
|
||||
[[ "${new_file,,}" == *.flac ]] || return 0
|
||||
# \x1f separator, NOT \t: tab is IFS whitespace, so consecutive tabs
|
||||
# (an empty field like $genres) collapse and shift every later field left —
|
||||
# GROUPING landing in GENRE etc. \x1f is non-whitespace, so empty fields
|
||||
# survive the read. Query by id: path queries are unreliable against the
|
||||
# mixed absolute/relative storage the beets 2.11 upgrade left in the DB.
|
||||
local raw
|
||||
raw=$(beet ls -f \
|
||||
$'$artist\x1f$albumartist\x1f$album\x1f$title\x1f$track\x1f$disc\x1f$year\x1f$genres\x1f$grouping' \
|
||||
"id:${existing_id}" 2>/dev/null | head -1)
|
||||
[[ -z "$raw" ]] && return 0
|
||||
local artist albumartist album title track disc year genre grouping
|
||||
IFS=$'\x1f' read -r artist albumartist album title track disc year genre grouping <<< "$raw"
|
||||
local tag
|
||||
for tag in ARTIST ALBUMARTIST ALBUM TITLE TRACKNUMBER DISCNUMBER DATE GENRE GROUPING; do
|
||||
metaflac --remove-tag="$tag" "$new_file" 2>/dev/null
|
||||
done
|
||||
[[ -n "$artist" ]] && metaflac --set-tag="ARTIST=$artist" "$new_file" 2>/dev/null
|
||||
[[ -n "$albumartist" ]] && metaflac --set-tag="ALBUMARTIST=$albumartist" "$new_file" 2>/dev/null
|
||||
[[ -n "$album" ]] && metaflac --set-tag="ALBUM=$album" "$new_file" 2>/dev/null
|
||||
[[ -n "$title" ]] && metaflac --set-tag="TITLE=$title" "$new_file" 2>/dev/null
|
||||
[[ -n "$track" && "$track" != "0" ]] && metaflac --set-tag="TRACKNUMBER=$track" "$new_file" 2>/dev/null
|
||||
[[ -n "$disc" && "$disc" != "0" ]] && metaflac --set-tag="DISCNUMBER=$disc" "$new_file" 2>/dev/null
|
||||
[[ -n "$year" && "$year" != "0" ]] && metaflac --set-tag="DATE=$year" "$new_file" 2>/dev/null
|
||||
[[ -n "$genre" ]] && metaflac --set-tag="GENRE=$genre" "$new_file" 2>/dev/null
|
||||
[[ -n "$grouping" ]] && metaflac --set-tag="GROUPING=$grouping" "$new_file" 2>/dev/null
|
||||
}
|
||||
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/replace-$(date +%Y%m%d-%H%M%S).log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
echo "[$(date -Iseconds)] === REPLACE (APPLY) scan=$SCAN_DIR ===" | tee -a "$LOG"
|
||||
# Radio prep (ReplayGain + autocue liq_* tags) BEFORE any import. Every
|
||||
# other import path preps first (run-playlist.sh, import-track.sh); this
|
||||
# one didn't, so replacement FLACs and leftover imports reached the library
|
||||
# unprepped for AzuraCast (found 2026-07-02: the July upgrade FLACs and a
|
||||
# 39-track leftover batch all lacked RG/autocue). Apply-mode only — dry
|
||||
# runs must not modify files.
|
||||
PREPPED=$(prep_audio "$SCAN_DIR" "$LOG" || echo 0)
|
||||
echo "[prep-audio] pre-tagged $PREPPED file(s) with ReplayGain + autocue" | tee -a "$LOG"
|
||||
else
|
||||
echo "[$(date -Iseconds)] === REPLACE (DRY RUN — pass --apply to act) scan=$SCAN_DIR ===" | tee -a "$LOG"
|
||||
fi
|
||||
|
||||
# A FLAC file always beats an MP3 file regardless of size. Among same-format
|
||||
# files, larger wins. Returns "new" or "existing".
|
||||
better() {
|
||||
local new="$1" existing="$2"
|
||||
local new_flac=0 ex_flac=0
|
||||
[[ "${new,,}" == *.flac ]] && new_flac=1
|
||||
[[ "${existing,,}" == *.flac ]] && ex_flac=1
|
||||
if [[ $new_flac -ne $ex_flac ]]; then
|
||||
if [[ $new_flac -eq 1 ]]; then echo "new"; else echo "existing"; fi
|
||||
return
|
||||
fi
|
||||
local ns es
|
||||
ns=$(stat -c '%s' "$new" 2>/dev/null || echo 0)
|
||||
es=$(stat -c '%s' "$existing" 2>/dev/null || echo 0)
|
||||
if [[ $ns -gt $es ]]; then echo "new"; else echo "existing"; fi
|
||||
}
|
||||
|
||||
# Read tag from FLAC or MP3 — return artist/album/title joined with newlines.
|
||||
read_tags() {
|
||||
local f="$1"
|
||||
case "${f,,}" in
|
||||
*.flac)
|
||||
local artist album title
|
||||
artist=$(metaflac --show-tag=ALBUMARTIST "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
[[ -z "$artist" ]] && artist=$(metaflac --show-tag=ARTIST "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
album=$(metaflac --show-tag=ALBUM "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
title=$(metaflac --show-tag=TITLE "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
printf "%s\n%s\n%s\n" "$artist" "$album" "$title"
|
||||
;;
|
||||
*.mp3)
|
||||
local artist album title
|
||||
artist=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TPE2[^:]*: //p' | head -1)
|
||||
[[ -z "$artist" ]] && artist=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TPE1[^:]*: //p' | head -1)
|
||||
album=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TALB[^:]*: //p' | head -1)
|
||||
title=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TIT2[^:]*: //p' | head -1)
|
||||
printf "%s\n%s\n%s\n" "$artist" "$album" "$title"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Beets queries (case-insensitive exact-substring on a field). Try progressively
|
||||
# looser matches until we find exactly one library track to compare against.
|
||||
# Echoes "<id>\x1f<path>" of the unique match; "" if no unique match found.
|
||||
beet_find_match() {
|
||||
local artist="$1" album="$2" title="$3"
|
||||
# Sanitize quotes
|
||||
artist="${artist//\"/}"; album="${album//\"/}"; title="${title//\"/}"
|
||||
|
||||
local fmt=$'$id\x1f$path'
|
||||
local hits
|
||||
# 1. Strictest: albumartist + album + title
|
||||
hits=$(beet ls -f "$fmt" \
|
||||
"albumartist:$artist" "album:$album" "title:$title" 2>/dev/null)
|
||||
[[ "$(echo "$hits" | wc -l)" == "1" && -n "$hits" ]] && { echo "$hits"; return; }
|
||||
|
||||
# 2. Drop album: maybe Bandcamp's "Artist | Album" doesn't match library's "Album"
|
||||
hits=$(beet ls -f "$fmt" \
|
||||
"albumartist:$artist" "title:$title" 2>/dev/null)
|
||||
[[ "$(echo "$hits" | wc -l)" == "1" && -n "$hits" ]] && { echo "$hits"; return; }
|
||||
|
||||
# 3. albumartist may be empty in old library entries — fall back to artist field
|
||||
hits=$(beet ls -f "$fmt" \
|
||||
"artist:$artist" "title:$title" 2>/dev/null)
|
||||
[[ "$(echo "$hits" | wc -l)" == "1" && -n "$hits" ]] && { echo "$hits"; return; }
|
||||
|
||||
# 4. Last resort: title alone — anchored exact match (case-insensitive),
|
||||
# NOT the substring match levels 1-3 use: a new track titled "Home"
|
||||
# must not "uniquely" match a library track titled "Coming Home Tonight"
|
||||
# and get it deleted. Only safe if exactly one library track carries
|
||||
# exactly this title.
|
||||
local escaped_title
|
||||
escaped_title=$(python3 -c \
|
||||
"import re,sys; print(re.escape(sys.argv[1]))" "$title" 2>/dev/null || echo "")
|
||||
if [[ -n "$escaped_title" ]]; then
|
||||
hits=$(beet ls -f "$fmt" \
|
||||
"title::(?i)^${escaped_title}\$" 2>/dev/null)
|
||||
[[ "$(echo "$hits" | wc -l)" == "1" && -n "$hits" ]] && { echo "$hits"; return; }
|
||||
fi
|
||||
|
||||
echo "" # no unique match
|
||||
}
|
||||
|
||||
REPLACED=0
|
||||
KEPT=0
|
||||
SKIPPED_NOMATCH=0
|
||||
|
||||
while IFS= read -r -d '' new_file; do
|
||||
# mapfile preserves empty lines (read with IFS=$'\n' would collapse them)
|
||||
mapfile -t _fields < <(read_tags "$new_file")
|
||||
artist="${_fields[0]:-}"
|
||||
album="${_fields[1]:-}"
|
||||
title="${_fields[2]:-}"
|
||||
if [[ -z "$artist" || -z "$title" ]]; then
|
||||
echo "[skip-untagged] $new_file" | tee -a "$LOG"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Find the matching library track via beets (progressive loosening)
|
||||
match=$(beet_find_match "$artist" "$album" "$title" | head -1)
|
||||
if [[ -z "$match" || "$match" != *$'\x1f'* ]]; then
|
||||
echo "[no-match] $artist - $album - $title (file: $new_file)" | tee -a "$LOG"
|
||||
SKIPPED_NOMATCH=$((SKIPPED_NOMATCH + 1))
|
||||
continue
|
||||
fi
|
||||
existing_id="${match%%$'\x1f'*}"
|
||||
existing_container="${match#*$'\x1f'}"
|
||||
|
||||
existing="${MUSIC_DATA_DIR:-/data/music}/Library${existing_container#/music}"
|
||||
if [[ ! -f "$existing" ]]; then
|
||||
echo "[stale-db] beets has $existing_container but file missing" | tee -a "$LOG"
|
||||
continue
|
||||
fi
|
||||
|
||||
winner=$(better "$new_file" "$existing")
|
||||
ns=$(numfmt --to=iec --suffix=B "$(stat -c '%s' "$new_file" 2>/dev/null || echo 0)")
|
||||
es=$(numfmt --to=iec --suffix=B "$(stat -c '%s' "$existing" 2>/dev/null || echo 0)")
|
||||
|
||||
if [[ "$winner" == "new" ]]; then
|
||||
echo "[REPLACE] $artist - $title ($es → $ns)" | tee -a "$LOG"
|
||||
echo " ex: $existing" | tee -a "$LOG"
|
||||
echo " new: $new_file" | tee -a "$LOG"
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
# If asked, copy canonical tags from the existing track to the new FLAC
|
||||
# BEFORE removing the existing entry. Used by upgrade-mp3-to-flac.sh so
|
||||
# the new FLAC inherits the MP3's spotify-retagged metadata (incl. the
|
||||
# GROUPING tag that drives playlist M3U regeneration).
|
||||
if [[ $COPY_TAGS -eq 1 ]]; then
|
||||
copy_tags_from_existing "$existing_id" "$new_file" >> "$LOG" 2>&1
|
||||
fi
|
||||
# Remove old library file + beets DB entry, then re-import the new one.
|
||||
# id query, not path: — see copy_tags_from_existing for why.
|
||||
beet remove -d -f "id:$existing_id" >> "$LOG" 2>&1
|
||||
beet import -q -s "$new_file" >> "$LOG" 2>&1
|
||||
fi
|
||||
REPLACED=$((REPLACED + 1))
|
||||
else
|
||||
echo "[KEEP] $artist - $title (existing $es ≥ new $ns)" | tee -a "$LOG"
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
rm -f "$new_file"
|
||||
fi
|
||||
KEPT=$((KEPT + 1))
|
||||
fi
|
||||
|
||||
done < <(find "$SCAN_DIR" -type f \( -iname "*.flac" -o -iname "*.mp3" \) -print0 2>/dev/null)
|
||||
|
||||
echo "" | tee -a "$LOG"
|
||||
echo "=== Summary: $REPLACED replaced, $KEPT existing-kept, $SKIPPED_NOMATCH no-match ===" | tee -a "$LOG"
|
||||
[[ $APPLY -eq 0 ]] && echo "DRY RUN — re-run with --apply to act" | tee -a "$LOG"
|
||||
|
||||
# After replaces, anything left in the scan dir is genuinely NEW (no library
|
||||
# match — not a duplicate of anything we already have). Import it as a new
|
||||
# track. Beets path templates will place it under /music/<albumartist>/...
|
||||
# alongside everything else.
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
remaining=$(find "$SCAN_DIR" -type f \( -iname "*.flac" -o -iname "*.mp3" \) 2>/dev/null | wc -l)
|
||||
if [[ "$remaining" -gt 0 ]]; then
|
||||
echo "[import-leftover] $remaining new file(s) under $SCAN_DIR — importing as fresh tracks" | tee -a "$LOG"
|
||||
beet import -q -s "$SCAN_DIR" >> "$LOG" 2>&1
|
||||
fi
|
||||
find "$SCAN_DIR" -mindepth 1 -type d -empty -delete 2>>"$LOG"
|
||||
echo "[cleanup] removed empty subfolders under $SCAN_DIR" | tee -a "$LOG"
|
||||
fi
|
||||
Reference in New Issue
Block a user