Stage 0: migrate pipeline scripts from /opt/sldl, scaffold repo
Moves all ~30 pipeline scripts, configs, and the vendored sldl binary into this repo (source /opt/sldl left untouched). Removes all docker exec/docker compose dependencies now that beets and sldl run in-process/as a subprocess of this container instead of via soulbeet/on-demand sldl containers. Replaces hardcoded host paths, Navidrome credentials, and Spotify credential sourcing with env-var-driven paths and shared credential loaders. Adds Dockerfile, entrypoint.sh, requirements.txt, docker-compose.snippet.yml, and the initial app DB schema.
This commit is contained in:
andrew
Executable
+280
@@ -0,0 +1,280 @@
|
||||
#!/bin/bash
|
||||
# import-track.sh
|
||||
#
|
||||
# Imports files from ${MUSIC_DATA_DIR:-/data/music}/import-me/ into your Navidrome library,
|
||||
# optionally tagging them with a playlist grouping.
|
||||
#
|
||||
# Drop files into ${MUSIC_DATA_DIR:-/data/music}/import-me/ (via SMB, web upload, copy, etc.)
|
||||
# then run this script to import them.
|
||||
#
|
||||
# Usage:
|
||||
# ./import-track.sh [playlist_name]
|
||||
# ./import-track.sh <file_or_subdir> [playlist_name]
|
||||
#
|
||||
# Examples:
|
||||
# ./import-track.sh # import everything in import-me/ (no playlist)
|
||||
# ./import-track.sh modular # import everything, tag all as 'modular'
|
||||
# ./import-track.sh my-track.flac # import just one file, no playlist
|
||||
# ./import-track.sh my-track.flac modular # import one file, tag as 'modular'
|
||||
# ./import-track.sh album_folder botanica # import folder, tag all as 'botanica'
|
||||
#
|
||||
# After successful import, files are moved out of import-me/ by beets.
|
||||
|
||||
set -euo pipefail
|
||||
# Commands that are allowed to exit non-zero must be wrapped explicitly via
|
||||
# `cmd && X=0 || X=$?` or `|| true`. Everything else aborts the import.
|
||||
|
||||
# ==== Paths ====
|
||||
IMPORT_ROOT=${MUSIC_DATA_DIR:-/data/music}/import-me
|
||||
DROPBOX_BASE=${MUSIC_DATA_DIR:-/data/music}/sldl-dropbox
|
||||
DOWNLOADS=${MUSIC_DATA_DIR:-/data/music}/downloads
|
||||
PLAYLISTS_DIR=${MUSIC_DATA_DIR:-/data/music}/playlists
|
||||
QUARANTINE_DIR=${MUSIC_DATA_DIR:-/data/music}/Songs/untagged
|
||||
LOG_DIR=${ALEMBIC_CONFIG_DIR:-/config}/logs
|
||||
LOG=${LOG_DIR}/manual-import-$(date +%Y%m%d).log
|
||||
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
|
||||
# ==== Parse arguments ====
|
||||
# Two modes:
|
||||
# 1. ./script.sh -> import all files in import-me/ root
|
||||
# 2. ./script.sh <playlist> -> import all files in import-me/ root with playlist tag
|
||||
# 3. ./script.sh <file> -> import specific file (no playlist)
|
||||
# 4. ./script.sh <file> <playlist> -> import specific file with playlist tag
|
||||
# Distinguish by checking if $1 is a valid path inside IMPORT_ROOT
|
||||
mkdir -p "$IMPORT_ROOT" "$LOG_DIR"
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
INPUT=""
|
||||
PLAYLIST_NAME=""
|
||||
elif [[ $# -eq 1 ]]; then
|
||||
# Could be either a playlist name OR a file path
|
||||
if [[ -e "${IMPORT_ROOT}/$1" ]]; then
|
||||
INPUT="$1"
|
||||
PLAYLIST_NAME=""
|
||||
else
|
||||
INPUT=""
|
||||
PLAYLIST_NAME="$1"
|
||||
fi
|
||||
elif [[ $# -eq 2 ]]; then
|
||||
INPUT="$1"
|
||||
PLAYLIST_NAME="$2"
|
||||
else
|
||||
cat <<EOF
|
||||
Usage: $0 [file_or_subdir] [playlist_name]
|
||||
|
||||
Place files in $IMPORT_ROOT/ first, then run this script.
|
||||
|
||||
Examples:
|
||||
$0 # import everything in import-me/
|
||||
$0 modular # import everything, tag as 'modular'
|
||||
$0 my-track.flac # import one file, no playlist
|
||||
$0 my-track.flac modular # import one file, tag as 'modular'
|
||||
$0 album_folder botanica # import folder, tag as 'botanica'
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log() {
|
||||
echo "[$(date -Iseconds)] $*" | tee -a "$LOG"
|
||||
}
|
||||
|
||||
log "=== Starting manual import ==="
|
||||
log "Input: ${INPUT:-<all in import-me/>}"
|
||||
log "Playlist: ${PLAYLIST_NAME:-<none>}"
|
||||
|
||||
# Resolve source path
|
||||
if [[ -n "$INPUT" ]]; then
|
||||
SOURCE="${IMPORT_ROOT}/${INPUT}"
|
||||
if [[ ! -e "$SOURCE" ]]; then
|
||||
log "ERROR: File or directory not found in import-me: $INPUT"
|
||||
log "Available items:"
|
||||
find "$IMPORT_ROOT" -mindepth 1 -maxdepth 1 -printf ' %f\n' | tee -a "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
SOURCE="$IMPORT_ROOT"
|
||||
# Check there's actually something to import
|
||||
if [[ -z "$(find "$IMPORT_ROOT" -type f \( -name "*.flac" -o -name "*.mp3" -o -name "*.wav" -o -name "*.m4a" \) 2>/dev/null)" ]]; then
|
||||
log "ERROR: No audio files found in $IMPORT_ROOT"
|
||||
log "Drop .flac, .mp3, .wav, or .m4a files into that folder first."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==== Determine destination ====
|
||||
if [[ -n "$PLAYLIST_NAME" ]]; then
|
||||
DEST_DIR="${DROPBOX_BASE}/${PLAYLIST_NAME}"
|
||||
else
|
||||
DEST_DIR="$DOWNLOADS"
|
||||
fi
|
||||
|
||||
mkdir -p "$DEST_DIR"
|
||||
|
||||
# ==== Convert WAV to FLAC (WAV doesn't support proper tagging) ====
|
||||
# Also parse "Artist - Title.wav" filename pattern to set basic FLAC tags,
|
||||
# since WAV files typically have no embedded metadata.
|
||||
log "Converting any WAV files to FLAC"
|
||||
|
||||
WAV_COUNT=0
|
||||
while IFS= read -r -d '' wavfile; do
|
||||
flacfile="${wavfile%.wav}.flac"
|
||||
if flac --silent --best --delete-input-file "$wavfile" -o "$flacfile" 2>>"$LOG"; then
|
||||
# Parse "Artist - Title.flac" from filename and set tags
|
||||
basename_noext=$(basename "$flacfile" .flac)
|
||||
if [[ "$basename_noext" == *" - "* ]]; then
|
||||
ARTIST="${basename_noext% - *}"
|
||||
TITLE="${basename_noext#* - }"
|
||||
metaflac \
|
||||
--set-tag="ARTIST=${ARTIST}" \
|
||||
--set-tag="TITLE=${TITLE}" \
|
||||
--set-tag="ALBUM=${TITLE}" \
|
||||
"$flacfile" 2>>"$LOG"
|
||||
log "Converted + tagged: $(basename "$flacfile") (Artist: $ARTIST, Title: $TITLE)"
|
||||
else
|
||||
log "Converted: $(basename "$flacfile") — WARNING: couldn't parse artist/title from filename"
|
||||
fi
|
||||
WAV_COUNT=$((WAV_COUNT + 1))
|
||||
else
|
||||
log "WARNING: Failed to convert $wavfile — leaving as WAV (will be untagged)"
|
||||
fi
|
||||
done < <(find "$SOURCE" -name "*.wav" -type f -print0 2>/dev/null)
|
||||
|
||||
[[ $WAV_COUNT -gt 0 ]] && log "Converted $WAV_COUNT WAV files to FLAC"
|
||||
|
||||
# If the user pointed us at a single .wav, that path no longer exists after
|
||||
# conversion (flac --delete-input-file) — repoint SOURCE at the converted
|
||||
# .flac so the tag checks, GROUPING, and move below operate on the real file.
|
||||
if [[ "$SOURCE" == *.wav && ! -e "$SOURCE" && -e "${SOURCE%.wav}.flac" ]]; then
|
||||
SOURCE="${SOURCE%.wav}.flac"
|
||||
log "Single-file WAV was converted; continuing with $(basename "$SOURCE")"
|
||||
fi
|
||||
|
||||
# ==== Quarantine files with missing essential tags ====
|
||||
log "Checking tags on audio files in $SOURCE"
|
||||
QUARANTINED=$(quarantine_untagged "$SOURCE" "$QUARANTINE_DIR" "$LOG" || echo 0)
|
||||
if [[ "$QUARANTINED" -gt 0 ]]; then
|
||||
log "Quarantined $QUARANTINED file(s) to $QUARANTINE_DIR — fix their tags and drop them back into $IMPORT_ROOT to retry"
|
||||
fi
|
||||
|
||||
# ==== Set ALBUMARTIST fallback so Navidrome doesn't coin ghost combined artists ====
|
||||
AA_SET=$(set_albumartist_fallback "$SOURCE" "$LOG" || echo 0)
|
||||
[[ "$AA_SET" -gt 0 ]] && log "Set ALBUMARTIST fallback on $AA_SET file(s) (primary artist before first separator)"
|
||||
|
||||
# Bail out if nothing survived the tag check
|
||||
if [[ -z "$(find "$SOURCE" -type f \( -iname "*.flac" -o -iname "*.mp3" -o -iname "*.m4a" \) 2>/dev/null)" ]]; then
|
||||
log "ERROR: no tagged audio files left to import after quarantine"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==== Tag files with playlist name BEFORE moving ====
|
||||
# Tagging in place in import-me/ ensures we only touch the files being
|
||||
# imported in this run, not any leftover files from previous runs.
|
||||
if [[ -n "$PLAYLIST_NAME" ]]; then
|
||||
log "Tagging files in import-me/ with GROUPING=$PLAYLIST_NAME"
|
||||
|
||||
FLAC_COUNT=0
|
||||
MP3_COUNT=0
|
||||
|
||||
# Scope of files to tag (dir or single file — find handles both)
|
||||
TAG_SCOPE="$SOURCE"
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
metaflac --remove-tag=GROUPING --set-tag="GROUPING=${PLAYLIST_NAME}" "$file" 2>>"$LOG" || true
|
||||
FLAC_COUNT=$((FLAC_COUNT + 1))
|
||||
done < <(find "$TAG_SCOPE" -name "*.flac" -type f -print0 2>/dev/null)
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
id3v2 --TIT1 "${PLAYLIST_NAME}" "$file" 2>>"$LOG" || true
|
||||
MP3_COUNT=$((MP3_COUNT + 1))
|
||||
done < <(find "$TAG_SCOPE" -name "*.mp3" -type f -print0 2>/dev/null)
|
||||
|
||||
log "Tagged $FLAC_COUNT FLAC and $MP3_COUNT MP3 files"
|
||||
fi
|
||||
|
||||
# ==== Move files from import-me to destination ====
|
||||
log "Moving from $SOURCE to $DEST_DIR"
|
||||
|
||||
if [[ -d "$SOURCE" ]] && [[ "$SOURCE" == "$IMPORT_ROOT" ]]; then
|
||||
# Moving all contents of import-me/ (but not the folder itself)
|
||||
shopt -s dotglob nullglob
|
||||
for item in "$SOURCE"/*; do
|
||||
[[ -e "$item" ]] && mv "$item" "$DEST_DIR/" 2>>"$LOG"
|
||||
done
|
||||
shopt -u dotglob nullglob
|
||||
elif [[ -d "$SOURCE" ]]; then
|
||||
# Moving a specific subdirectory
|
||||
mv "$SOURCE" "$DEST_DIR/" 2>>"$LOG"
|
||||
else
|
||||
# Moving a single file
|
||||
mv "$SOURCE" "$DEST_DIR/" 2>>"$LOG"
|
||||
fi
|
||||
|
||||
# ==== Clean up any .incomplete files before beets touches them ====
|
||||
log "Cleaning up .incomplete files in $DEST_DIR"
|
||||
find "$DEST_DIR" -name "*.incomplete" -type f -delete 2>>"$LOG" || true
|
||||
|
||||
# ==== Pre-tag with loudgain (ReplayGain) + autocue (liq_* cue/overlay tags) ====
|
||||
# Done here, on the staging copy, so Navidrome/AzuraCast see fully-prepped files
|
||||
# the moment beets moves them into /music.
|
||||
PREPPED=$(prep_audio "$DEST_DIR" "$LOG" || echo 0)
|
||||
log "Pre-tagged $PREPPED file(s) with ReplayGain + autocue"
|
||||
|
||||
# ==== Strip MusicBrainz / release-identification tags ====
|
||||
# Manual imports often come from MB-tagged sources (Apple Music rips, Soulseek,
|
||||
# etc.); Navidrome's BFR scanner uses those tags to fragment album cards, so
|
||||
# one tagged track imported into an otherwise-clean album gets split off.
|
||||
# Mirrors the strip step that spotify-retag.py runs in the playlist flow.
|
||||
STRIPPED=$(strip_mb_tags "$DEST_DIR" "$LOG")
|
||||
[[ "$STRIPPED" -gt 0 ]] && log "Stripped MB/release tags from $STRIPPED file(s)"
|
||||
|
||||
# ==== Trigger beets import (in-process, same container) ====
|
||||
log "Running beets import on $DEST_DIR"
|
||||
# Beets can return non-zero on duplicate skip — not fatal.
|
||||
BEETS_EXIT=0
|
||||
beet import -q -s "$DEST_DIR" >> "$LOG" 2>&1 || BEETS_EXIT=$?
|
||||
log "beets import finished with exit code $BEETS_EXIT"
|
||||
|
||||
# ==== Regenerate M3U if playlist was specified ====
|
||||
if [[ -n "$PLAYLIST_NAME" ]]; then
|
||||
mkdir -p "$PLAYLISTS_DIR"
|
||||
M3U_OUT="${PLAYLISTS_DIR}/${PLAYLIST_NAME}.m3u8"
|
||||
log "Regenerating M3U at $M3U_OUT"
|
||||
|
||||
BEET_OUTPUT=$(beet ls -f '$path' "grouping:${PLAYLIST_NAME}" 2>>"$LOG" || true)
|
||||
# grep -c '^' exits 1 on empty input, which would trip set -e and kill the
|
||||
# whole run silently right here. Count lines a way that's safe on no-match.
|
||||
if [[ -z "$BEET_OUTPUT" ]]; then
|
||||
TRACK_COUNT=0
|
||||
else
|
||||
TRACK_COUNT=$(printf '%s\n' "$BEET_OUTPUT" | wc -l)
|
||||
fi
|
||||
|
||||
if [[ "$TRACK_COUNT" -gt 0 ]]; then
|
||||
{
|
||||
echo "#EXTM3U"
|
||||
echo "$BEET_OUTPUT"
|
||||
} > "$M3U_OUT"
|
||||
log "M3U updated with $TRACK_COUNT tracks"
|
||||
else
|
||||
log "WARNING: no tracks found with grouping:$PLAYLIST_NAME — M3U not updated"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==== Trigger Navidrome scan (through the share-health gate) ====
|
||||
# navidrome-scan.sh refuses to trigger a scan while the QNAP share is down or
|
||||
# truncated. With ND_SCANNER_PURGEMISSING=always, an ungated full scan against
|
||||
# a bad mount makes Navidrome purge the entire library (the 2026-06-01 wipe) —
|
||||
# this script used to fire a raw startScan curl with no such check.
|
||||
log "Triggering Navidrome scan via navidrome-scan.sh (share-health gated)"
|
||||
${PIPELINE_DIR:-/app/pipeline}/lib/navidrome-scan.sh >> "$LOG" 2>&1 || log "WARNING: navidrome-scan.sh exited non-zero"
|
||||
|
||||
log "=== Finished manual import ==="
|
||||
log ""
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user