Stage 0: migrate pipeline scripts from /opt/sldl, scaffold repo
Moves all ~30 pipeline scripts, configs, and the vendored sldl binary into this repo (source /opt/sldl left untouched). Removes all docker exec/docker compose dependencies now that beets and sldl run in-process/as a subprocess of this container instead of via soulbeet/on-demand sldl containers. Replaces hardcoded host paths, Navidrome credentials, and Spotify credential sourcing with env-var-driven paths and shared credential loaders. Adds Dockerfile, entrypoint.sh, requirements.txt, docker-compose.snippet.yml, and the initial app DB schema.
This commit is contained in:
andrew
Executable
+215
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
backfill-buy-url.py — stamp the Bandcamp purchase URL onto library files for
|
||||
albums/tracks already imported from a Bandcamp purchase.
|
||||
|
||||
The forward path (sync-bandcamp.py) tags new purchases at download time. This
|
||||
one-shot covers everything bought *before* that change landed. It:
|
||||
|
||||
1. Pulls the current Bandcamp collection (cookies auth) and builds an index
|
||||
keyed by (album_artist, album) -> release URL. (Bandcamp's embedded ALBUM
|
||||
tag matches the collection item_title exactly, and ALBUMARTIST matches the
|
||||
band name — verified against the library.)
|
||||
2. Walks the FLAC library, reads ALBUMARTIST/ALBUM, and where a match exists,
|
||||
writes the COMMERCIAL_INFORMATION Vorbis comment (AzuraCast maps this to
|
||||
the buy_url custom field — see sync-bandcamp.py BUY_URL_TAG).
|
||||
3. Optionally tells AzuraCast to reprocess each touched file so the custom
|
||||
field repopulates without waiting for the periodic media scan.
|
||||
|
||||
Dry-run by default. Stdlib only (mutagen NOT required — uses metaflac).
|
||||
|
||||
Usage:
|
||||
backfill-buy-url.py --cookies F --state F [--library DIR] [--apply]
|
||||
[--azuracast-key ID:SECRET] [--azuracast-base URL]
|
||||
[--station N] [--force]
|
||||
"""
|
||||
import sys, os, re, json, time, html, argparse, subprocess
|
||||
import urllib.request
|
||||
from http.cookiejar import MozillaCookieJar
|
||||
from pathlib import Path
|
||||
|
||||
COLLECTION_ITEMS_URL = "https://bandcamp.com/api/fancollection/1/collection_items"
|
||||
UA = "Mozilla/5.0 (X11; Linux x86_64) backfill-buy-url/1.0"
|
||||
BUY_URL_TAG = "COMMERCIAL_INFORMATION"
|
||||
|
||||
|
||||
# ---- Bandcamp collection ---------------------------------------------------
|
||||
def opener(jar):
|
||||
return urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))
|
||||
|
||||
|
||||
def get_fan_id(username, jar):
|
||||
req = urllib.request.Request(f"https://bandcamp.com/{username}",
|
||||
headers={"User-Agent": UA})
|
||||
page = html.unescape(opener(jar).open(req, timeout=60).read().decode("utf-8", "replace"))
|
||||
m = re.search(r'"fan_id"\s*:\s*"?(\d+)', page)
|
||||
if not m:
|
||||
sys.exit("[backfill] could not find fan_id (cookies expired / wrong user?)")
|
||||
return int(m.group(1))
|
||||
|
||||
|
||||
def fetch_collection(fan_id, jar):
|
||||
items = []
|
||||
tok = "9999999999::a::"
|
||||
while True:
|
||||
body = json.dumps({"fan_id": fan_id, "older_than_token": tok, "count": 100}).encode()
|
||||
req = urllib.request.Request(
|
||||
COLLECTION_ITEMS_URL, data=body,
|
||||
headers={"User-Agent": UA, "Content-Type": "application/json",
|
||||
"Accept": "application/json", "Referer": "https://bandcamp.com/"})
|
||||
data = json.loads(opener(jar).open(req, timeout=60).read())
|
||||
page = data.get("items", [])
|
||||
items.extend(page)
|
||||
if not data.get("more_available") or not page:
|
||||
break
|
||||
nt = data.get("last_token")
|
||||
if not nt or nt == tok:
|
||||
break
|
||||
tok = nt
|
||||
return items
|
||||
|
||||
|
||||
# ---- matching --------------------------------------------------------------
|
||||
def norm(s):
|
||||
return re.sub(r"\s+", " ", (s or "").strip()).casefold()
|
||||
|
||||
|
||||
def build_index(items):
|
||||
"""(album_artist, album) -> url. Index both the bare title and the
|
||||
"<title> - Single" form (single-track purchases get that ALBUM tag)."""
|
||||
idx = {}
|
||||
for it in items:
|
||||
url = it.get("item_url")
|
||||
band = it.get("band_name")
|
||||
title = it.get("item_title")
|
||||
if not (url and band and title):
|
||||
continue
|
||||
for album in (title, f"{title} - Single"):
|
||||
idx.setdefault((norm(band), norm(album)), url)
|
||||
return idx
|
||||
|
||||
|
||||
def flac_tag(path, name):
|
||||
r = subprocess.run(["metaflac", f"--show-tag={name}", path],
|
||||
capture_output=True, text=True)
|
||||
for line in r.stdout.splitlines():
|
||||
if "=" in line:
|
||||
return line.split("=", 1)[1]
|
||||
return ""
|
||||
|
||||
|
||||
def set_flac_tag(path, name, value):
|
||||
subprocess.run(["metaflac", f"--remove-tag={name}", path], capture_output=True)
|
||||
return subprocess.run(["metaflac", f"--set-tag={name}={value}", path],
|
||||
capture_output=True).returncode == 0
|
||||
|
||||
|
||||
# ---- AzuraCast reprocess ---------------------------------------------------
|
||||
def az_reprocess(base, key, station, host_paths, log):
|
||||
"""Find each host path in AzuraCast and batch-reprocess so the custom
|
||||
field repopulates. host_path -> relative AzuraCast path is library-root
|
||||
stripped."""
|
||||
root = f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library/"
|
||||
want = {p[len(root):] for p in host_paths if p.startswith(root)}
|
||||
headers = {"X-API-Key": key, "Accept": "application/json"}
|
||||
|
||||
def get(path):
|
||||
req = urllib.request.Request(base + path, headers=headers)
|
||||
return json.loads(urllib.request.urlopen(req, timeout=60).read())
|
||||
|
||||
unique_ids, page = [], 1
|
||||
while True:
|
||||
d = get(f"/api/station/{station}/files?per_page=500&page={page}")
|
||||
for row in d["rows"]:
|
||||
if row["path"] in want:
|
||||
unique_ids.append(row["unique_id"])
|
||||
if page >= d["total_pages"]:
|
||||
break
|
||||
page += 1
|
||||
if not unique_ids:
|
||||
log(" no matching AzuraCast files to reprocess")
|
||||
return
|
||||
body = json.dumps({"do": "reprocess", "files": unique_ids}).encode()
|
||||
req = urllib.request.Request(base + f"/api/station/{station}/files/batch",
|
||||
data=body, method="PUT",
|
||||
headers={**headers, "Content-Type": "application/json"})
|
||||
urllib.request.urlopen(req, timeout=120).read()
|
||||
log(f" queued reprocess for {len(unique_ids)} AzuraCast file(s)")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--cookies", required=True)
|
||||
ap.add_argument("--state", required=True, help="state.json (for reference/logging)")
|
||||
ap.add_argument("--library", default=f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library")
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--force", action="store_true",
|
||||
help="overwrite an existing buy-link tag")
|
||||
ap.add_argument("--azuracast-key", default=None)
|
||||
ap.add_argument("--azuracast-base", default="https://admin.ephemeral.club")
|
||||
ap.add_argument("--station", type=int, default=1)
|
||||
ap.add_argument("--username", default=None,
|
||||
help="Bandcamp username; default read from config.env if present")
|
||||
args = ap.parse_args()
|
||||
|
||||
user = args.username
|
||||
if not user:
|
||||
cfg = f"{os.environ.get('ALEMBIC_CONFIG_DIR', '/config')}/pipeline/bandcamp/config.env"
|
||||
if os.path.exists(cfg):
|
||||
for line in Path(cfg).read_text().splitlines():
|
||||
if line.startswith("BANDCAMP_USERNAME="):
|
||||
user = line.split("=", 1)[1].strip().strip('"')
|
||||
if not user:
|
||||
sys.exit("[backfill] --username required (no config.env BANDCAMP_USERNAME)")
|
||||
|
||||
print(f"[backfill] mode={'APPLY' if args.apply else 'DRY RUN'} user={user!r}")
|
||||
jar = MozillaCookieJar(args.cookies)
|
||||
jar.load(ignore_discard=True, ignore_expires=True)
|
||||
fan_id = get_fan_id(user, jar)
|
||||
items = fetch_collection(fan_id, jar)
|
||||
idx = build_index(items)
|
||||
print(f"[backfill] collection: {len(items)} items, {len(idx)} match keys")
|
||||
|
||||
flacs = list(Path(args.library).rglob("*.flac"))
|
||||
print(f"[backfill] scanning {len(flacs)} FLAC files...\n")
|
||||
|
||||
matched = tagged = already = unmatched = 0
|
||||
touched = []
|
||||
for p in flacs:
|
||||
sp = str(p)
|
||||
albumartist = flac_tag(sp, "ALBUMARTIST") or flac_tag(sp, "ARTIST")
|
||||
album = flac_tag(sp, "ALBUM")
|
||||
url = idx.get((norm(albumartist), norm(album)))
|
||||
if not url:
|
||||
unmatched += 1
|
||||
continue
|
||||
matched += 1
|
||||
existing = flac_tag(sp, BUY_URL_TAG)
|
||||
if existing and not args.force:
|
||||
already += 1
|
||||
continue
|
||||
rel = sp[len(args.library):].lstrip("/")
|
||||
print(f" + {rel}\n -> {url}")
|
||||
if args.apply:
|
||||
if set_flac_tag(sp, BUY_URL_TAG, url):
|
||||
tagged += 1
|
||||
touched.append(sp)
|
||||
else:
|
||||
print(" ! metaflac write failed")
|
||||
else:
|
||||
tagged += 1
|
||||
touched.append(sp)
|
||||
|
||||
print(f"\n[backfill] matched={matched} {'tagged' if args.apply else 'would-tag'}={tagged}"
|
||||
f" already-had={already} unmatched-files={unmatched}")
|
||||
|
||||
if args.apply and touched and args.azuracast_key:
|
||||
print("[backfill] telling AzuraCast to reprocess touched files...")
|
||||
az_reprocess(args.azuracast_base, args.azuracast_key, args.station,
|
||||
touched, lambda m: print(m))
|
||||
elif not args.apply:
|
||||
print("[backfill] re-run with --apply to write (add --azuracast-key to reprocess)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
# One-shot backfill: rewrite tags from Spotify for every track currently in the
|
||||
# beets library that has a grouping:<playlist> tag matching one of our configs.
|
||||
#
|
||||
# After the rewrite, sync beets DB from file tags, then move files to clean
|
||||
# paths based on the new tags, then trigger a Navidrome rescan.
|
||||
|
||||
set -u
|
||||
|
||||
CONFIGS="${ALEMBIC_CONFIG_DIR:-/config}/pipeline"
|
||||
LOG="${ALEMBIC_CONFIG_DIR:-/config}/logs/backfill-$(date +%Y%m%d-%H%M%S).log"
|
||||
RETAG="${PIPELINE_DIR:-/app/pipeline}/lib/spotify-retag.py"
|
||||
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
echo "[$(date -Iseconds)] === Backfill start ===" | tee -a "$LOG"
|
||||
|
||||
TOTAL_TRACKS=0
|
||||
TOTAL_PLAYLISTS=0
|
||||
|
||||
for conf in "$CONFIGS"/*.conf; do
|
||||
[[ "$conf" == "$CONFIGS/_template.conf" ]] && continue
|
||||
playlist=$(basename "$conf" .conf)
|
||||
|
||||
url=$(sed -n 's/^input *= *//p' "$conf" | tr -d ' ')
|
||||
cid=$(sed -n 's/^spotify-id *= *//p' "$conf" | tr -d ' ')
|
||||
csec=$(sed -n 's/^spotify-secret *= *//p' "$conf" | tr -d ' ')
|
||||
|
||||
if [[ -z "$url" || -z "$cid" || -z "$csec" ]]; then
|
||||
echo "[$playlist] SKIP: missing url/creds in conf" | tee -a "$LOG"
|
||||
continue
|
||||
fi
|
||||
|
||||
# beets runs in-process in this same container, so its reported paths
|
||||
# need no container-to-host translation.
|
||||
paths=$(beet ls -f '$path' "grouping:$playlist" 2>/dev/null)
|
||||
count=$(echo -n "$paths" | grep -c '^' || true)
|
||||
|
||||
if [[ "$count" -eq 0 ]]; then
|
||||
echo "[$playlist] no tracks tagged in beets — skip" | tee -a "$LOG"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[$playlist] retagging $count tracks via Spotify" | tee -a "$LOG"
|
||||
if echo "$paths" | SPOTIFY_CLIENT_ID="$cid" SPOTIFY_CLIENT_SECRET="$csec" \
|
||||
python3 "$RETAG" "$url" - >> "$LOG" 2>&1; then
|
||||
echo "[$playlist] retag OK" | tee -a "$LOG"
|
||||
TOTAL_PLAYLISTS=$((TOTAL_PLAYLISTS + 1))
|
||||
TOTAL_TRACKS=$((TOTAL_TRACKS + count))
|
||||
else
|
||||
echo "[$playlist] retag FAILED — see $LOG" | tee -a "$LOG"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[$(date -Iseconds)] === Retagged $TOTAL_TRACKS tracks across $TOTAL_PLAYLISTS playlists ===" | tee -a "$LOG"
|
||||
|
||||
echo "[$(date -Iseconds)] Syncing beets DB from file tags (beet update)..." | tee -a "$LOG"
|
||||
beet update -F path 2>&1 | tail -20 >> "$LOG"
|
||||
|
||||
echo "[$(date -Iseconds)] Moving files into clean paths (beet move)..." | tee -a "$LOG"
|
||||
beet move 2>&1 | tail -5 >> "$LOG"
|
||||
|
||||
NAVIDROME_ENV="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/navidrome/admin.env"
|
||||
[ -f "$NAVIDROME_ENV" ] && source "$NAVIDROME_ENV"
|
||||
ND_BASE="${ND_BASE:-http://navidrome:4533}"
|
||||
ND_USER="${ND_USER:-andrew}"
|
||||
ND_PASS="${ND_PASS:-}"
|
||||
|
||||
echo "[$(date -Iseconds)] Triggering Navidrome full scan" | tee -a "$LOG"
|
||||
curl -s -G "$ND_BASE/rest/startScan.view" \
|
||||
--data-urlencode "u=$ND_USER" \
|
||||
--data-urlencode "p=$ND_PASS" \
|
||||
--data-urlencode 'v=1.16.0' \
|
||||
--data-urlencode 'c=backfill' \
|
||||
--data-urlencode 'f=json' \
|
||||
--data-urlencode 'fullScan=true' >> "$LOG"
|
||||
|
||||
echo "[$(date -Iseconds)] === Backfill complete. Log: $LOG ===" | tee -a "$LOG"
|
||||
Executable
+191
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
build-fingerprint-index.py — build/refresh a Chromaprint fingerprint index of
|
||||
the beets library at $ALEMBIC_CONFIG_DIR/beets/fingerprints.db.
|
||||
|
||||
Reads paths from `beet ls -f '$id|||$path'` (beets now runs in-process in this
|
||||
same container, so no container-to-host path translation is needed — the path
|
||||
beets reports is already valid in this container's own filesystem view).
|
||||
Skips entries whose (path, mtime, size) is unchanged. Parallel fpcalc workers
|
||||
keep this under ~10 min for a 3k-track library.
|
||||
|
||||
Used by import-dj-collection.py to dedup incoming DJ files against the existing
|
||||
library via acoustid.compare_fingerprints().
|
||||
|
||||
Usage:
|
||||
build-fingerprint-index.py [--workers N] [--limit N]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import multiprocessing
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
|
||||
DB_PATH = f"{ALEMBIC_CONFIG_DIR}/beets/fingerprints.db"
|
||||
DEFAULT_WORKERS = 8
|
||||
|
||||
|
||||
def init_db(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS fingerprints (
|
||||
beets_id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
mtime REAL NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
duration INTEGER NOT NULL,
|
||||
fingerprint TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_fp_path ON fingerprints(path);
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def enumerate_library() -> list[tuple[int, str]]:
|
||||
"""Return [(beets_id, path), ...] for every track in the library."""
|
||||
cmd = ["beet", "ls", "-f", "$id|||$path"]
|
||||
out = subprocess.run(cmd, capture_output=True, text=True, check=True).stdout
|
||||
items: list[tuple[int, str]] = []
|
||||
for line in out.splitlines():
|
||||
line = line.rstrip("\r")
|
||||
if "|||" not in line:
|
||||
continue
|
||||
id_str, path = line.split("|||", 1)
|
||||
try:
|
||||
beets_id = int(id_str)
|
||||
except ValueError:
|
||||
continue
|
||||
items.append((beets_id, path))
|
||||
return items
|
||||
|
||||
|
||||
def fpcalc(host_path: str) -> tuple[int, str] | None:
|
||||
"""Run fpcalc -raw on a file. Returns (duration_seconds, fingerprint_csv)."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["fpcalc", "-raw", host_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return None
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
duration = 0
|
||||
fingerprint = ""
|
||||
for line in proc.stdout.splitlines():
|
||||
if line.startswith("DURATION="):
|
||||
try:
|
||||
duration = int(line.split("=", 1)[1])
|
||||
except ValueError:
|
||||
pass
|
||||
elif line.startswith("FINGERPRINT="):
|
||||
fingerprint = line.split("=", 1)[1]
|
||||
if not fingerprint:
|
||||
return None
|
||||
return duration, fingerprint
|
||||
|
||||
|
||||
def worker(job: tuple[int, str, float, int]) -> tuple[int, str, float, int, int, str] | None:
|
||||
beets_id, host_path, mtime, size = job
|
||||
result = fpcalc(host_path)
|
||||
if result is None:
|
||||
return None
|
||||
duration, fingerprint = result
|
||||
return beets_id, host_path, mtime, size, duration, fingerprint
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--workers", type=int, default=DEFAULT_WORKERS)
|
||||
ap.add_argument("--limit", type=int, default=0, help="process at most N new files (0=all)")
|
||||
ap.add_argument("--verbose", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
init_db(conn)
|
||||
|
||||
print(f"[index] enumerating library via beets...", flush=True)
|
||||
library = enumerate_library()
|
||||
print(f"[index] library has {len(library)} tracks", flush=True)
|
||||
|
||||
existing: dict[int, tuple[str, float, int]] = {}
|
||||
for row in conn.execute("SELECT beets_id, path, mtime, size FROM fingerprints"):
|
||||
existing[row[0]] = (row[1], row[2], row[3])
|
||||
|
||||
jobs: list[tuple[int, str, float, int]] = []
|
||||
missing_count = 0
|
||||
for beets_id, host_path in library:
|
||||
try:
|
||||
st = os.stat(host_path)
|
||||
except FileNotFoundError:
|
||||
missing_count += 1
|
||||
continue
|
||||
cur = existing.get(beets_id)
|
||||
if cur is not None and cur[0] == host_path and abs(cur[1] - st.st_mtime) < 1 and cur[2] == st.st_size:
|
||||
continue
|
||||
jobs.append((beets_id, host_path, st.st_mtime, st.st_size))
|
||||
|
||||
print(f"[index] {len(jobs)} new/changed tracks to fingerprint "
|
||||
f"({len(library) - len(jobs) - missing_count} already current, {missing_count} missing on disk)", flush=True)
|
||||
if args.limit > 0:
|
||||
jobs = jobs[:args.limit]
|
||||
print(f"[index] --limit {args.limit} applied; processing {len(jobs)}", flush=True)
|
||||
if not jobs:
|
||||
conn.close()
|
||||
return 0
|
||||
|
||||
purge_ids = set(existing.keys()) - {b for b, *_ in library}
|
||||
if purge_ids:
|
||||
conn.executemany("DELETE FROM fingerprints WHERE beets_id = ?", [(i,) for i in purge_ids])
|
||||
conn.commit()
|
||||
print(f"[index] purged {len(purge_ids)} stale rows", flush=True)
|
||||
|
||||
start = time.time()
|
||||
done = 0
|
||||
failed = 0
|
||||
batch: list[tuple] = []
|
||||
with multiprocessing.Pool(args.workers) as pool:
|
||||
for result in pool.imap_unordered(worker, jobs, chunksize=4):
|
||||
done += 1
|
||||
if result is None:
|
||||
failed += 1
|
||||
else:
|
||||
batch.append(result)
|
||||
if len(batch) >= 50:
|
||||
conn.executemany(
|
||||
"INSERT OR REPLACE INTO fingerprints "
|
||||
"(beets_id, path, mtime, size, duration, fingerprint) VALUES (?,?,?,?,?,?)",
|
||||
batch,
|
||||
)
|
||||
conn.commit()
|
||||
batch.clear()
|
||||
if done % 100 == 0 or args.verbose:
|
||||
elapsed = time.time() - start
|
||||
rate = done / elapsed if elapsed > 0 else 0
|
||||
print(f"[index] {done}/{len(jobs)} ({rate:.1f}/s, {failed} failed)", flush=True)
|
||||
if batch:
|
||||
conn.executemany(
|
||||
"INSERT OR REPLACE INTO fingerprints "
|
||||
"(beets_id, path, mtime, size, duration, fingerprint) VALUES (?,?,?,?,?,?)",
|
||||
batch,
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
total = conn.execute("SELECT COUNT(*) FROM fingerprints").fetchone()[0]
|
||||
conn.close()
|
||||
elapsed = time.time() - start
|
||||
print(f"[index] done in {elapsed:.0f}s. wrote {done - failed} rows, {failed} failed. "
|
||||
f"index now has {total} entries.", flush=True)
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
clean-sldl-index.py — for each playlist's _sldl.m3u8, drop entries whose
|
||||
track is no longer anywhere in the beets library. sldl uses the m3u as its
|
||||
"already downloaded" skip list, so anything left in there is skipped on
|
||||
the next playlist run. By removing entries for files we deleted (placeholders,
|
||||
quarantined junk), we let sldl retry them fresh.
|
||||
|
||||
Conservative matching — we only drop a m3u line if no library track even
|
||||
loosely matches its (artist, title). If we drop too aggressively, sldl
|
||||
re-downloads good tracks we still have. If we keep too liberally, deleted
|
||||
placeholders linger in the m3u — but that's a smaller problem.
|
||||
|
||||
Usage:
|
||||
clean-sldl-index.py # dry run
|
||||
clean-sldl-index.py --apply # rewrite m3us in place (.bak.<ts> backup)
|
||||
"""
|
||||
import sys, re, subprocess, argparse, os, time, shutil
|
||||
from pathlib import Path
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
DROPBOX = f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/sldl-dropbox"
|
||||
|
||||
|
||||
def normalize(s):
|
||||
"""Lowercase, strip parens content + feat/ft suffix, collapse whitespace.
|
||||
Keep ALL alphanumeric (including Unicode/Japanese) — strip only spaces and
|
||||
common punctuation."""
|
||||
s = s.lower()
|
||||
s = re.sub(r"\s*\([^)]*\)", "", s)
|
||||
s = re.sub(r"\s*\[[^\]]*\]", "", s)
|
||||
s = re.sub(r"\s*feat\.?\s.*", "", s)
|
||||
s = re.sub(r"\s*ft\.?\s.*", "", s)
|
||||
s = re.sub(r"[\s\-_,;:&'\"!?.()\[\]/]+", "", s)
|
||||
return s
|
||||
|
||||
|
||||
def fetch_beets_index():
|
||||
"""Return list of (norm_artist_title, norm_artist, norm_title) tuples for
|
||||
every track in beets, normalized for comparison."""
|
||||
r = subprocess.run(
|
||||
["beet", "ls", "-f", "$artist‖$title"],
|
||||
capture_output=True, text=True, timeout=120
|
||||
)
|
||||
out = []
|
||||
for line in r.stdout.splitlines():
|
||||
if "‖" not in line:
|
||||
continue
|
||||
artist, title = line.split("‖", 1)
|
||||
n_a = normalize(artist)
|
||||
n_t = normalize(title)
|
||||
out.append((n_a + n_t, n_a, n_t))
|
||||
return out
|
||||
|
||||
|
||||
def parse_m3u_line(line):
|
||||
"""Extract (artist, title) from a m3u filename like 'Artist - Title.ext'.
|
||||
If the line is FAIL-prefixed or comment, return (None, None)."""
|
||||
s = line.rstrip("\n")
|
||||
if not s or s.startswith("#"):
|
||||
return None, None
|
||||
stem = re.sub(r"\.[a-zA-Z0-9]{1,5}$", "", s)
|
||||
if " - " in stem:
|
||||
# First " - " split: typical "Artist - Title"
|
||||
a, _, t = stem.partition(" - ")
|
||||
return a.strip(), t.strip()
|
||||
return None, stem
|
||||
|
||||
|
||||
def m3u_matches_beets(line, beets_idx):
|
||||
"""Return True if some beets track plausibly matches this m3u line."""
|
||||
artist, title = parse_m3u_line(line)
|
||||
if artist is None and title is None:
|
||||
return True # comment/blank — keep
|
||||
n_a = normalize(artist or "")
|
||||
n_t = normalize(title or "")
|
||||
n_full = n_a + n_t
|
||||
|
||||
if not n_full:
|
||||
return True # nothing to match against — be safe and keep
|
||||
|
||||
for beets_full, beets_a, beets_t in beets_idx:
|
||||
if not beets_full:
|
||||
continue
|
||||
# Cheap exact full match
|
||||
if n_full == beets_full:
|
||||
return True
|
||||
# Mutual substring (handles multi-artist credits added to one side)
|
||||
if n_full in beets_full or beets_full in n_full:
|
||||
return True
|
||||
# Title-only match if artist sides are very different (multi-artist case)
|
||||
if n_t and beets_t and (n_t == beets_t or n_t in beets_t or beets_t in n_t):
|
||||
# require some artist overlap (substring either way) to avoid false hits
|
||||
if n_a and beets_a and (n_a in beets_a or beets_a in n_a):
|
||||
return True
|
||||
# Romaji ↔ Japanese cases: the m3u uses Spotify's romaji artist
|
||||
# ("natori") but the file has the original kanji ("なとり"). Exact
|
||||
# title match with a non-trivial length is enough.
|
||||
if n_t == beets_t and len(n_t) > 6:
|
||||
return True
|
||||
# Last-ditch fuzzy: very similar overall strings
|
||||
if len(n_full) > 6 and len(beets_full) > 6:
|
||||
ratio = SequenceMatcher(None, n_full, beets_full).quick_ratio()
|
||||
if ratio > 0.85:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
print(f"[clean-index] mode={'APPLY' if args.apply else 'DRY RUN'}")
|
||||
print("[clean-index] loading beets library index...")
|
||||
beets_idx = fetch_beets_index()
|
||||
print(f"[clean-index] indexed {len(beets_idx)} tracks")
|
||||
|
||||
# Safety guard: an empty index means `beet ls` failed (missing/locked DB,
|
||||
# bad config) — NOT that the library is empty. Without this check,
|
||||
# --apply would drop every entry from every _sldl.m3u8 and the next
|
||||
# playlist runs would re-download the entire library from Soulseek.
|
||||
if not beets_idx:
|
||||
print("[clean-index] ABORT: beets index came back empty "
|
||||
"(beet ls failed) — refusing to touch any m3u",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
|
||||
total_kept = 0
|
||||
total_dropped = 0
|
||||
total_files = 0
|
||||
|
||||
for m3u in sorted(Path(DROPBOX).glob("*/_sldl.m3u8")):
|
||||
playlist = m3u.parent.name
|
||||
with open(m3u) as f:
|
||||
lines = f.readlines()
|
||||
kept, dropped = [], []
|
||||
for line in lines:
|
||||
if m3u_matches_beets(line, beets_idx):
|
||||
kept.append(line)
|
||||
else:
|
||||
dropped.append(line)
|
||||
print(f"\n[{playlist}] {len(kept)} kept, {len(dropped)} dropped")
|
||||
for d in dropped[:8]:
|
||||
print(f" DROP {d.rstrip()}")
|
||||
if len(dropped) > 8:
|
||||
print(f" ...and {len(dropped) - 8} more")
|
||||
|
||||
total_kept += len(kept)
|
||||
total_dropped += len(dropped)
|
||||
total_files += 1
|
||||
|
||||
if args.apply and dropped:
|
||||
backup = f"{m3u}.bak.{int(time.time())}"
|
||||
shutil.copy(m3u, backup)
|
||||
with open(m3u, "w") as f:
|
||||
f.writelines(kept)
|
||||
print(f" [{playlist}] rewritten; backup at {backup}")
|
||||
|
||||
print(f"\n[clean-index] {total_files} m3us, total {total_kept} kept, {total_dropped} dropped")
|
||||
if not args.apply:
|
||||
print("[clean-index] DRY RUN — re-run with --apply to rewrite the m3us")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+321
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
clear-bad-genres.py — normalize or blank GENRE tags so every file ends up
|
||||
with a clean "; "-joined list of whitelist entries (max 3 parts).
|
||||
|
||||
Two-tier behavior per file:
|
||||
1. NORMALIZE: if every part of the GENRE value canonicalizes to a
|
||||
whitelist entry (after splitting on ; , / or " - " and resolving aliases
|
||||
like "Rnb" → "R&B", "Hip Hop" → "Hip-Hop"), and there are ≤3 parts,
|
||||
REWRITE the file's GENRE in canonical "Display1; Display2" form. This
|
||||
fixes formatting issues like "Dance;Pop" (no space) → "Dance; Pop"
|
||||
without losing the data.
|
||||
2. BLANK: if any part doesn't map to a whitelist canonical, OR there are
|
||||
too many parts, BLANK the GENRE field. Run spotify-genre.py --apply
|
||||
--force afterwards to refill what Spotify can.
|
||||
|
||||
Catches:
|
||||
- Mojibake (Cyrillic-as-ASCII like '5:B@>==0O <C7K:0')
|
||||
- Wrong delimiters: comma ('Pop, Rock'), slash ('Indie Rock/Rock Pop'),
|
||||
space-dash-space ('Dance - Electro - Vocal'), missing space after
|
||||
semicolon ('Dance;Pop')
|
||||
- Mixed-delimiter chains ('Pop, Rock; Alternative')
|
||||
- Bracketed values ('[Drum n Bass]')
|
||||
- Over-tagging: more than 3 parts (matches spotify-genre.py's MAX_GENRES)
|
||||
- Spelling variants: 'Rnb' → 'R&B', 'Hip Hop' → 'Hip-Hop',
|
||||
'Drum n Bass' / 'Drum and Bass' → 'Drum & Bass', 'K Pop' → 'K-Pop'
|
||||
- Niche values not in the whitelist
|
||||
|
||||
Already-clean tracks are left untouched (zero file writes).
|
||||
|
||||
Usage:
|
||||
clear-bad-genres.py # dry run, show planned normalize+blank ops
|
||||
clear-bad-genres.py --apply # actually rewrite/blank
|
||||
"""
|
||||
import sys, os, re, argparse, subprocess
|
||||
from mutagen import File as MFile
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError, TCON
|
||||
from mutagen.flac import FLAC
|
||||
from mutagen.mp4 import MP4
|
||||
from mutagen.oggopus import OggOpus
|
||||
from mutagen.oggvorbis import OggVorbis
|
||||
|
||||
|
||||
# OGG/OPUS use Vorbis-comment-style tags (lowercase keys, multi-value), same
|
||||
# shape as FLAC. Treat them as a family so the read/write paths are shared.
|
||||
VORBIS_LIKE = (FLAC, OggOpus, OggVorbis)
|
||||
|
||||
ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
|
||||
WHITELIST_FILE = f"{ALEMBIC_CONFIG_DIR}/pipeline/genres-whitelist.txt"
|
||||
|
||||
|
||||
MAX_PARTS = 3 # mirrors spotify-genre.py's MAX_GENRES
|
||||
SEPARATOR = "; "
|
||||
|
||||
# Map normalized variant → normalized whitelist entry. Spotify and various
|
||||
# Soulseek uploaders use spelling variants of the same genre — we want all of
|
||||
# them to land on a single canonical whitelist string.
|
||||
ALIASES = {
|
||||
"rnb": "rb", # Rnb → R&B
|
||||
"randb": "rb", # R and B
|
||||
"hiphop": "hiphop", # noop after normalization
|
||||
"hiphopalt": "hiphop",
|
||||
"hphop": "hiphop",
|
||||
"drumnbass": "drumbass", # Drum n Bass → Drum & Bass
|
||||
"drumandbass": "drumbass", # Drum and Bass
|
||||
"dnb": "drumbass", # DnB / D&B
|
||||
"kpop": "kpop",
|
||||
"jpop": "jpop",
|
||||
"trance": "trance",
|
||||
"idm": "idm",
|
||||
"edm": "edm",
|
||||
"ukgarage": "ukgarage",
|
||||
"rocknroll": "rockroll", # Rock n Roll → Rock & Roll
|
||||
"rockandroll": "rockroll",
|
||||
"popfilm": "popfilm",
|
||||
"popsoundtrack": "popfilm",
|
||||
}
|
||||
|
||||
|
||||
def normed(s):
|
||||
"""lowercase + strip whitespace/hyphens/underscores/&/slashes/dots."""
|
||||
return re.sub(r"[\s\-_&/.]+", "", s).lower()
|
||||
|
||||
|
||||
def load_whitelist():
|
||||
"""Return dict: normalized form → canonical display string from the file."""
|
||||
out = {}
|
||||
with open(WHITELIST_FILE) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
out[normed(line)] = line
|
||||
return out
|
||||
|
||||
|
||||
# Splits on: ; , / space-dash-space (any of these = multi-genre delimiter)
|
||||
SPLIT_RE = re.compile(r"\s+-\s+|[;,/]")
|
||||
|
||||
|
||||
def canonicalize(value, wl):
|
||||
"""Try to parse value into ≤MAX_PARTS canonical whitelist entries.
|
||||
|
||||
Returns:
|
||||
(parts_canonical, all_mapped) where:
|
||||
parts_canonical = list of whitelist display strings (deduped, original
|
||||
order), possibly empty
|
||||
all_mapped = True iff every parsed part landed on a whitelist entry
|
||||
AND the part count is within MAX_PARTS
|
||||
"""
|
||||
raw_parts = [p.strip(" \t[]()<>") for p in SPLIT_RE.split(value)]
|
||||
raw_parts = [p for p in raw_parts if p]
|
||||
if not raw_parts or len(raw_parts) > MAX_PARTS:
|
||||
return [], False
|
||||
|
||||
seen = set()
|
||||
out = []
|
||||
for p in raw_parts:
|
||||
n = normed(p)
|
||||
# Try alias first, then direct whitelist hit
|
||||
canon_norm = ALIASES.get(n, n)
|
||||
display = wl.get(canon_norm)
|
||||
if not display:
|
||||
return [], False
|
||||
if display not in seen:
|
||||
out.append(display)
|
||||
seen.add(display)
|
||||
return out, True
|
||||
|
||||
|
||||
def assess(value, wl):
|
||||
"""Return ('clean'|'rewrite'|'blank', new_value).
|
||||
|
||||
- 'clean' : current value already canonical, no write needed
|
||||
- 'rewrite' : all parts map cleanly; rewrite file as new_value
|
||||
- 'blank' : at least one part is junk; blank the field
|
||||
"""
|
||||
if not value or not value.strip():
|
||||
return "clean", None
|
||||
canon_parts, ok = canonicalize(value, wl)
|
||||
if not ok:
|
||||
return "blank", None
|
||||
new_value = SEPARATOR.join(canon_parts)
|
||||
if new_value == value:
|
||||
return "clean", None
|
||||
return "rewrite", new_value
|
||||
|
||||
|
||||
def is_genre_locked(m):
|
||||
"""True if the file has GENRE_LOCK=1 (set by fix-genre.sh).
|
||||
|
||||
Manually-overridden genres often won't match the whitelist (e.g. 'Bass'
|
||||
isn't in the whitelist but G Jones genuinely needs it). The lock means
|
||||
'this is intentional, leave it alone'."""
|
||||
if not m or not m.tags:
|
||||
return False
|
||||
try:
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
val = (m.tags.get("genre_lock") or m.tags.get("GENRE_LOCK") or [None])[0]
|
||||
return val == "1"
|
||||
if isinstance(m, MP4):
|
||||
return False # fix-genre.sh doesn't set lock on MP4 yet
|
||||
for frame in m.tags.getall("TXXX"):
|
||||
if frame.desc.upper() == "GENRE_LOCK" and frame.text and frame.text[0] == "1":
|
||||
return True
|
||||
except (AttributeError, IndexError, TypeError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def get_genre(filepath):
|
||||
"""Return (mutagen_obj, joined_genre_string).
|
||||
|
||||
Vorbis comments (FLAC) and ID3v2.4 TCON both allow multiple GENRE values
|
||||
per file. Beets joins them with '; ' for display. We mirror that so a file
|
||||
with 8 separate GENRE tags is correctly seen as an 8-part chain (and gets
|
||||
flagged as over-tagged), not just whatever the first tag happened to be.
|
||||
"""
|
||||
try:
|
||||
m = MFile(filepath)
|
||||
except Exception:
|
||||
return None, None
|
||||
if not m or not m.tags:
|
||||
return m, None
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
vals = m.tags.get("genre") or []
|
||||
return m, ("; ".join(str(v) for v in vals) if vals else None)
|
||||
if isinstance(m, MP4):
|
||||
vals = m.tags.get("©gen") or []
|
||||
return m, ("; ".join(str(v) for v in vals) if vals else None)
|
||||
if "TCON" in m.tags and m.tags["TCON"].text:
|
||||
vals = m.tags["TCON"].text
|
||||
return m, "; ".join(str(v) for v in vals)
|
||||
return m, None
|
||||
|
||||
|
||||
def clear_genre(filepath, m):
|
||||
try:
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
if "genre" in m.tags:
|
||||
del m.tags["genre"]
|
||||
m.save()
|
||||
elif isinstance(m, MP4):
|
||||
if "©gen" in m.tags:
|
||||
del m.tags["©gen"]
|
||||
m.save()
|
||||
else:
|
||||
tags = ID3(filepath)
|
||||
tags.delall("TCON")
|
||||
tags.save(filepath)
|
||||
return True
|
||||
except (AttributeError, ID3NoHeaderError, Exception):
|
||||
return False
|
||||
|
||||
|
||||
def write_genre(filepath, m, value):
|
||||
"""Overwrite GENRE with a single string value (replacing any multi-value
|
||||
set the file might have had — that's the whole point of this rewrite)."""
|
||||
try:
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
m.tags["genre"] = [value]
|
||||
m.save()
|
||||
elif isinstance(m, MP4):
|
||||
m.tags["©gen"] = [value]
|
||||
m.save()
|
||||
else:
|
||||
try:
|
||||
tags = ID3(filepath)
|
||||
except ID3NoHeaderError:
|
||||
tags = ID3()
|
||||
tags.delall("TCON")
|
||||
tags.add(TCON(encoding=3, text=value))
|
||||
tags.save(filepath)
|
||||
return True
|
||||
except (AttributeError, ID3NoHeaderError, Exception):
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
wl = load_whitelist()
|
||||
print(f"[clear-bad-genres] whitelist has {len(wl)} entries")
|
||||
|
||||
# beets now runs in-process in this same container, so the paths it
|
||||
# reports are already valid here directly — no container-to-host
|
||||
# translation needed.
|
||||
r = subprocess.run(
|
||||
["beet", "ls", "-f", "$path"],
|
||||
capture_output=True, text=True, timeout=120
|
||||
)
|
||||
paths = [p for p in r.stdout.splitlines() if p.strip()]
|
||||
print(f"[clear-bad-genres] {len(paths)} tracks to scan")
|
||||
|
||||
rewritten = blanked = failed = locked = 0
|
||||
rewrite_examples = {} # original value -> (sample filename, canonical form)
|
||||
blank_examples = {} # original value -> sample filename
|
||||
|
||||
for p in paths:
|
||||
if not os.path.exists(p):
|
||||
continue
|
||||
m, current = get_genre(p)
|
||||
if m is None or current is None:
|
||||
continue
|
||||
if is_genre_locked(m):
|
||||
locked += 1
|
||||
continue
|
||||
action, new_value = assess(current, wl)
|
||||
if action == "clean":
|
||||
continue
|
||||
|
||||
if action == "rewrite":
|
||||
if current not in rewrite_examples:
|
||||
rewrite_examples[current] = (os.path.basename(p), new_value)
|
||||
if args.apply:
|
||||
if write_genre(p, m, new_value):
|
||||
rewritten += 1
|
||||
else:
|
||||
failed += 1
|
||||
else:
|
||||
rewritten += 1
|
||||
else: # blank
|
||||
if current not in blank_examples:
|
||||
blank_examples[current] = os.path.basename(p)
|
||||
if args.apply:
|
||||
if clear_genre(p, m):
|
||||
blanked += 1
|
||||
else:
|
||||
failed += 1
|
||||
else:
|
||||
blanked += 1
|
||||
|
||||
verb = "rewrote" if args.apply else "would-rewrite"
|
||||
bverb = "blanked" if args.apply else "would-blank"
|
||||
print(f"\n[clear-bad-genres] {verb}: {rewritten} tracks, {bverb}: {blanked} tracks, "
|
||||
f"locked-skipped: {locked}, failed: {failed}")
|
||||
|
||||
if rewrite_examples:
|
||||
print(f"\nNormalize-rewrites ({len(rewrite_examples)} distinct):")
|
||||
for old, (sample, new) in sorted(rewrite_examples.items())[:50]:
|
||||
print(f" {old!r:40s} → {new!r:30s} e.g. {sample}")
|
||||
if len(rewrite_examples) > 50:
|
||||
print(f" ...and {len(rewrite_examples)-50} more")
|
||||
|
||||
if blank_examples:
|
||||
print(f"\nBlanked (will be refilled by spotify-genre) ({len(blank_examples)} distinct):")
|
||||
for value, sample in sorted(blank_examples.items())[:50]:
|
||||
print(f" {value!r:60s} e.g. {sample}")
|
||||
if len(blank_examples) > 50:
|
||||
print(f" ...and {len(blank_examples)-50} more")
|
||||
|
||||
if args.apply and (rewritten or blanked):
|
||||
print("\n[clear-bad-genres] now run:")
|
||||
print(" beet update")
|
||||
if blanked:
|
||||
print(" spotify-genre.py --apply --force")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
# Convert m4a files in-place: AAC -> MP3 (V0 VBR), ALAC -> FLAC (lossless).
|
||||
# Preserves metadata + embedded cover art. Deletes original on success.
|
||||
set -u
|
||||
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/convert-m4a-$(date +%Y%m%d-%H%M%S).log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
log() { echo "[$(date -Iseconds)] $*" | tee -a "$LOG"; }
|
||||
|
||||
OK_AAC=0
|
||||
OK_ALAC=0
|
||||
FAIL=0
|
||||
|
||||
while IFS= read -r -d '' f; do
|
||||
codec=$(ffprobe -v error -select_streams a:0 -show_entries stream=codec_name -of csv=p=0 "$f" 2>/dev/null | tr -d ',')
|
||||
base="${f%.m4a}"
|
||||
case "$codec" in
|
||||
aac)
|
||||
out="${base}.mp3"
|
||||
if [[ -e "$out" ]]; then log "SKIP-EXISTS: $out"; FAIL=$((FAIL+1)); continue; fi
|
||||
if ffmpeg -nostdin -hide_banner -loglevel error -i "$f" \
|
||||
-map 0 -c:v copy -c:a libmp3lame -q:a 0 -id3v2_version 3 \
|
||||
"$out" 2>>"$LOG"; then
|
||||
rm -f "$f"
|
||||
log "AAC→MP3: $out"
|
||||
OK_AAC=$((OK_AAC+1))
|
||||
else
|
||||
log "FAIL-AAC: $f"
|
||||
rm -f "$out" 2>/dev/null
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
;;
|
||||
alac)
|
||||
out="${base}.flac"
|
||||
if [[ -e "$out" ]]; then log "SKIP-EXISTS: $out"; FAIL=$((FAIL+1)); continue; fi
|
||||
if ffmpeg -nostdin -hide_banner -loglevel error -i "$f" \
|
||||
-map 0 -c:v copy -c:a flac -compression_level 8 \
|
||||
"$out" 2>>"$LOG"; then
|
||||
rm -f "$f"
|
||||
log "ALAC→FLAC: $out"
|
||||
OK_ALAC=$((OK_ALAC+1))
|
||||
else
|
||||
log "FAIL-ALAC: $f"
|
||||
rm -f "$out" 2>/dev/null
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
log "SKIP-OTHER ($codec): $f"
|
||||
;;
|
||||
esac
|
||||
done < <(find ${MUSIC_DATA_DIR:-/data/music}/Library -type f -iname "*.m4a" -print0 2>/dev/null)
|
||||
|
||||
log ""
|
||||
log "=== AAC→MP3: $OK_AAC | ALAC→FLAC: $OK_ALAC | Failed: $FAIL ==="
|
||||
log "Log: $LOG"
|
||||
Executable
+461
@@ -0,0 +1,461 @@
|
||||
#!/bin/bash
|
||||
# Walk the beets library for duplicates using three passes:
|
||||
#
|
||||
# Pass 1 — numbered siblings: find every *.N.ext in beets where *.ext also
|
||||
# exists on disk in the same directory; keep the better file.
|
||||
#
|
||||
# Pass 2 — case-insensitive beet-level dedup: pull all tracks from beets,
|
||||
# group by lowercase (albumartist, album, title); any group with
|
||||
# 2+ members is a duplicate set — keep FLAC > MP3 > largest file.
|
||||
#
|
||||
# Pass 3 — normalized beet-level dedup: same as Pass 2 but the key strips
|
||||
# all non-alphanumeric characters before grouping. Catches dupes
|
||||
# whose tags differ only by separator/punctuation/feat. credit
|
||||
# ("Vol. 7" vs ", Vol. 7"; "Artist1; Artist2" vs "Artist1/Artist2").
|
||||
#
|
||||
# "Best" is always: FLAC > MP3/other, then largest file within the same format.
|
||||
#
|
||||
# DRY RUN by default. Pass --apply to actually delete.
|
||||
|
||||
set -u
|
||||
APPLY=0
|
||||
[[ "${1:-}" == "--apply" ]] && APPLY=1
|
||||
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$(date +%Y%m%d-%H%M%S).log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
log() { echo "$*" | tee -a "$LOG"; }
|
||||
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
log "[$(date -Iseconds)] === DEDUP (APPLY) ==="
|
||||
else
|
||||
log "[$(date -Iseconds)] === DEDUP (DRY RUN — pass --apply to delete) ==="
|
||||
fi
|
||||
|
||||
# Counters: tracked via log grep at the end (avoids bash subshell pitfalls).
|
||||
# Functions write KEEP/DELETE lines with consistent prefixes; summary greps them.
|
||||
|
||||
# Convert container path (/music/...) to host path (${MUSIC_DATA_DIR:-/data/music}/Library/...)
|
||||
host_path() { echo "${MUSIC_DATA_DIR:-/data/music}/Library${1#/music}"; }
|
||||
|
||||
SEP=$'\x1c' # ASCII file separator — safe with any music metadata
|
||||
|
||||
# Rank a file: lower = better. FLAC > MP3 > other (WAV/etc.); clean filename
|
||||
# > sync-conflict artifact (-DESKTOP-, (1), .1.); extended/club mix > radio
|
||||
# edit at same format; then larger wins.
|
||||
#
|
||||
# WAV ranks below MP3 deliberately: WAV has no real tag container and in this
|
||||
# library only appears as a sldl download glitch. Keeping a large WAV over a
|
||||
# tagged MP3 of the same track is never what we want.
|
||||
rank_file() {
|
||||
local hp="$1"
|
||||
local size; size=$(stat -c '%s' "$hp" 2>/dev/null || echo 0)
|
||||
local ext_score=0
|
||||
[[ "${hp,,}" == *.flac ]] && ext_score=2
|
||||
[[ "${hp,,}" == *.mp3 ]] && ext_score=1
|
||||
local dirty_score=0
|
||||
# (N) copy-artifact check is capped at 2 digits: "(1).flac" is a Windows
|
||||
# copy artifact, but "(2014).flac" is a title ending in a year — marking
|
||||
# those dirty inverted the keep preference toward their .1.flac siblings.
|
||||
if [[ "$hp" =~ -DESKTOP-[A-Z0-9]+ ]] \
|
||||
|| [[ "$hp" =~ \([0-9]{1,2}\)\.[a-z]+$ ]] \
|
||||
|| [[ "$hp" =~ \.[0-9]+\.[a-z]+$ ]]; then
|
||||
dirty_score=1
|
||||
fi
|
||||
local extended_score=0
|
||||
shopt -s nocasematch
|
||||
if [[ "$hp" =~ (extended|club\ mix|dj\ edit|dj\ mix|long\ version|original\ mix) ]]; then
|
||||
extended_score=1
|
||||
fi
|
||||
shopt -u nocasematch
|
||||
echo $((10000000000 - ext_score * 1000000000000 + dirty_score * 100000000000 - extended_score * 50000000000 - size))
|
||||
}
|
||||
|
||||
# process_group LABEL ENTRY [ENTRY ...]
|
||||
# ENTRY is either "<beets-id>\x1c<container-path>" (a track in beets — deleted
|
||||
# via `beet remove id:N`) or a bare host path (a ghost file on disk only;
|
||||
# rm-only to delete).
|
||||
#
|
||||
# Why id queries, not path queries: the beets 2.11 upgrade (~2026-06-19) left
|
||||
# the DB with MIXED path storage — pre-upgrade items store absolute
|
||||
# /music/... paths, post-upgrade imports store library-relative paths. No
|
||||
# single path:/path:: query form matches both populations (this is what made
|
||||
# every dedup DELETE a silent no-op for two weeks). Item ids are storage-
|
||||
# format-proof.
|
||||
process_group() {
|
||||
local label="$1"; shift
|
||||
|
||||
local ranked=""
|
||||
local hp id entry
|
||||
for entry in "$@"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
if [[ "$entry" == *"$SEP"* ]]; then
|
||||
id="${entry%%"$SEP"*}"
|
||||
hp=$(host_path "${entry#*"$SEP"}")
|
||||
else
|
||||
id=""
|
||||
hp="$entry"
|
||||
fi
|
||||
[[ -f "$hp" ]] || continue
|
||||
local score; score=$(rank_file "$hp")
|
||||
ranked+="${score}${SEP}${id}${SEP}${hp}"$'\n'
|
||||
done
|
||||
|
||||
[[ -z "$ranked" ]] && return 1 # no files on disk — skip
|
||||
|
||||
log ""
|
||||
log "--- $label ---"
|
||||
|
||||
local first=1
|
||||
while IFS="$SEP" read -r _sk id hp; do
|
||||
[[ -z "$hp" ]] && continue
|
||||
local size_h; size_h=$(numfmt --to=iec --suffix=B "$(stat -c '%s' "$hp" 2>/dev/null || echo 0)")
|
||||
if [[ $first -eq 1 ]]; then
|
||||
log " KEEP ($size_h) $hp"
|
||||
first=0
|
||||
else
|
||||
log " DELETE ($size_h) $hp"
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
if [[ -n "$id" ]]; then
|
||||
# In beets — beet remove -d removes from DB + disk.
|
||||
beet remove -d -f "id:${id}" >> "$LOG" 2>&1
|
||||
else
|
||||
# Ghost file — only on disk; just rm
|
||||
rm -f "$hp" 2>>"$LOG"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done < <(echo "$ranked" | grep -v '^$' | sort -n)
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# Pass 1: numbered siblings
|
||||
# Find every *.N.ext in beets where *.ext also exists on disk.
|
||||
# ============================================================
|
||||
log ""
|
||||
log "[$(date -Iseconds)] === Pass 1: numbered siblings ==="
|
||||
|
||||
# Dump id + container path from beets once. beets normalizes $path for
|
||||
# display (always /music/-prefixed) even though the DB stores a mix of
|
||||
# absolute and relative — so the display paths are safe to compare/convert,
|
||||
# and the ids are what we hand to beet remove/move.
|
||||
beet ls -f "\$id${SEP}\$path" 2>/dev/null > /tmp/beets-id-paths.txt
|
||||
cut -d"$SEP" -f2- /tmp/beets-id-paths.txt > /tmp/beets-all-paths.txt
|
||||
|
||||
while IFS="$SEP" read -r numbered_id numbered_cp; do
|
||||
[[ -z "$numbered_cp" ]] && continue
|
||||
|
||||
ext="${numbered_cp##*.}"
|
||||
canonical_cp=$(echo "$numbered_cp" | sed -E "s/\.[0-9]+\.${ext}$/.${ext}/")
|
||||
[[ "$canonical_cp" == "$numbered_cp" ]] && continue # no .N. found
|
||||
|
||||
host_numbered=$(host_path "$numbered_cp")
|
||||
host_canonical=$(host_path "$canonical_cp")
|
||||
|
||||
[[ -f "$host_numbered" ]] || continue
|
||||
[[ -f "$host_canonical" ]] || continue
|
||||
|
||||
# If the canonical is also in beets, Pass 2 handles this pair via beet remove -d.
|
||||
# Skip here to avoid a partial rm that leaves a stale beets DB entry.
|
||||
grep -qxF "$canonical_cp" /tmp/beets-all-paths.txt && continue
|
||||
|
||||
score_numbered=$(rank_file "$host_numbered")
|
||||
score_canonical=$(rank_file "$host_canonical")
|
||||
|
||||
if [[ $score_canonical -le $score_numbered ]]; then
|
||||
# Canonical wins: delete numbered (in beets), keep canonical (ghost)
|
||||
process_group "P1 numbered-sibling: ${numbered_cp##/music/}" \
|
||||
"$host_canonical" "${numbered_id}${SEP}${numbered_cp}"
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
# canonical is now a ghost file; import it into beets
|
||||
beet import -q -s "${canonical_cp}" >> "$LOG" 2>&1
|
||||
fi
|
||||
else
|
||||
# Numbered wins (canonical is ghost): rm the ghost, then beet move to rename
|
||||
# the numbered file to the canonical path (id query — see process_group).
|
||||
process_group "P1 numbered-sibling: ${numbered_cp##/music/}" \
|
||||
"${numbered_id}${SEP}${numbered_cp}" "$host_canonical"
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
beet move "id:${numbered_id}" >> "$LOG" 2>&1
|
||||
fi
|
||||
fi
|
||||
done < <(grep -E '\.[0-9]+\.(flac|mp3|m4a|ogg|opus|wav)$' /tmp/beets-id-paths.txt)
|
||||
|
||||
P1_GROUPS=$(grep -c '^--- P1 ' "$LOG" 2>/dev/null; true)
|
||||
log ""
|
||||
log "Pass 1 complete: $P1_GROUPS numbered-sibling groups"
|
||||
|
||||
# ============================================================
|
||||
# Pass 2: case-insensitive beet-level dedup
|
||||
# Re-fetch track list so Pass 1 deletions are reflected.
|
||||
# ============================================================
|
||||
log ""
|
||||
log "[$(date -Iseconds)] === Pass 2: case-insensitive title/album/artist dedup ==="
|
||||
|
||||
beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
|
||||
2>/dev/null > /tmp/all-tracks.txt
|
||||
|
||||
awk -F"${SEP}" -v sep="$SEP" '
|
||||
# Sentinel titles carry zero identity — multiple distinct tracks can land on
|
||||
# the same value when spotify-retag misses and the Soulseek source had a
|
||||
# blank/placeholder title. Grouping by sentinel title = deleting different
|
||||
# tracks that happened to share a placeholder. Common offenders:
|
||||
# "" fully empty title
|
||||
# "[unknown]" "unknown" sldl/beets fallback
|
||||
# "track 1" .. "track NN" stock CD-rip placeholder
|
||||
function is_sentinel_title(t, lt) {
|
||||
lt = tolower(t)
|
||||
gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
|
||||
if (lt == "") return 1
|
||||
if (lt == "[unknown]" || lt == "unknown") return 1
|
||||
if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
|
||||
return 0
|
||||
}
|
||||
{
|
||||
if (is_sentinel_title($4)) next
|
||||
key = tolower($2) sep tolower($3) sep tolower($4)
|
||||
paths[key] = paths[key] $1 sep $5 "\n"
|
||||
count[key]++
|
||||
}
|
||||
END {
|
||||
for (k in paths) {
|
||||
if (count[k] >= 2) print "===" k "\n" paths[k] "---"
|
||||
}
|
||||
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt
|
||||
|
||||
GROUP_KEY=""
|
||||
GROUP_PATHS=()
|
||||
|
||||
flush_p2_group() {
|
||||
[[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
|
||||
process_group "P2 case-insensitive: $GROUP_KEY" "${GROUP_PATHS[@]}"
|
||||
GROUP_PATHS=()
|
||||
}
|
||||
|
||||
while IFS= read -r line; do
|
||||
if [[ "$line" == ===* ]]; then
|
||||
flush_p2_group
|
||||
GROUP_KEY="${line#===}"
|
||||
GROUP_PATHS=()
|
||||
elif [[ "$line" == "---" ]]; then
|
||||
flush_p2_group
|
||||
elif [[ -n "$line" ]]; then
|
||||
GROUP_PATHS+=("$line")
|
||||
fi
|
||||
done < /tmp/dups-grouped.txt
|
||||
flush_p2_group # flush last group
|
||||
|
||||
P2_GROUPS=$(grep -c '^--- P2 ' "$LOG" 2>/dev/null; true)
|
||||
log ""
|
||||
log "Pass 2 complete: $P2_GROUPS case-insensitive dup groups"
|
||||
|
||||
# ============================================================
|
||||
# Pass 3: normalized-key dedup (strip non-alphanumerics)
|
||||
# Re-fetch track list so Pass 2 deletions are reflected.
|
||||
# ============================================================
|
||||
log ""
|
||||
log "[$(date -Iseconds)] === Pass 3: normalized title/album/artist dedup ==="
|
||||
|
||||
beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
|
||||
2>/dev/null > /tmp/all-tracks.txt
|
||||
|
||||
# Normalize: lowercase + strip everything except a-z 0-9. Group by the
|
||||
# concatenation. Suppresses dupe pairs already caught by Pass 2 (same exact
|
||||
# key) by also tracking the raw key per group and skipping single-key groups.
|
||||
awk -F"${SEP}" -v sep="$SEP" '
|
||||
function norm(s, t) {
|
||||
t = tolower(s)
|
||||
gsub(/[^a-z0-9]/, "", t)
|
||||
return t
|
||||
}
|
||||
function is_sentinel_title(t, lt) {
|
||||
lt = tolower(t)
|
||||
gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
|
||||
if (lt == "") return 1
|
||||
if (lt == "[unknown]" || lt == "unknown") return 1
|
||||
if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
|
||||
return 0
|
||||
}
|
||||
{
|
||||
if (is_sentinel_title($4)) next
|
||||
raw = tolower($2) sep tolower($3) sep tolower($4)
|
||||
na = norm($2); nb = norm($3); nc = norm($4)
|
||||
# Title is the only field guaranteed unique per track. If it normalizes
|
||||
# to empty (all unicode/punctuation), refuse to group — would cause false
|
||||
# positives across unrelated unicode-titled tracks. A 1-char normalized
|
||||
# title is fine as long as artist+album also normalize non-empty
|
||||
# (covers "ID"/"OP"/single-letter tracks; rejects all-unicode titles
|
||||
# that collapse to "").
|
||||
if (length(nc) == 0) next
|
||||
if (length(nc) < 3 && (length(na) == 0 || length(nb) == 0)) next
|
||||
k = na sep nb sep nc
|
||||
paths[k] = paths[k] $1 sep $5 "\n"
|
||||
count[k]++
|
||||
raws[k][raw] = 1
|
||||
}
|
||||
END {
|
||||
for (k in paths) {
|
||||
if (count[k] < 2) continue
|
||||
# Skip groups that collapse to a single raw key — those were Pass 2 dups.
|
||||
raw_count = 0
|
||||
for (r in raws[k]) raw_count++
|
||||
if (raw_count < 2) continue
|
||||
print "===" k "\n" paths[k] "---"
|
||||
}
|
||||
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt
|
||||
|
||||
GROUP_KEY=""
|
||||
GROUP_PATHS=()
|
||||
|
||||
flush_p3_group() {
|
||||
[[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
|
||||
process_group "P3 normalized: $GROUP_KEY" "${GROUP_PATHS[@]}"
|
||||
GROUP_PATHS=()
|
||||
}
|
||||
|
||||
while IFS= read -r line; do
|
||||
if [[ "$line" == ===* ]]; then
|
||||
flush_p3_group
|
||||
GROUP_KEY="${line#===}"
|
||||
GROUP_PATHS=()
|
||||
elif [[ "$line" == "---" ]]; then
|
||||
flush_p3_group
|
||||
elif [[ -n "$line" ]]; then
|
||||
GROUP_PATHS+=("$line")
|
||||
fi
|
||||
done < /tmp/dups-grouped.txt
|
||||
flush_p3_group # flush last group
|
||||
|
||||
P3_GROUPS=$(grep -c '^--- P3 ' "$LOG" 2>/dev/null; true)
|
||||
log ""
|
||||
log "Pass 3 complete: $P3_GROUPS normalized dup groups"
|
||||
|
||||
# ============================================================
|
||||
# Pass 4: cross-album dedup (album-agnostic key)
|
||||
# Catches cases where the same track exists once as a single and once on a
|
||||
# compilation/Beatport-style album — the album field always disagrees, so
|
||||
# Pass 2/3 miss them. Keys on (artist_norm, title_base_norm) where title_base
|
||||
# strips trailing mix-version suffixes ("(Extended Mix)", " - Radio Edit",
|
||||
# bare " Extended Mix" with double-space, etc.).
|
||||
#
|
||||
# Allowlist: artists who legitimately have the same track on multiple albums
|
||||
# (live + studio + remix etc.) — these are SKIPPED in Pass 4. Add to
|
||||
# CROSS_ALBUM_KEEP_RE or the album-pattern skip below if needed.
|
||||
# ============================================================
|
||||
log ""
|
||||
log "[$(date -Iseconds)] === Pass 4: cross-album dedup (artist+title-base) ==="
|
||||
|
||||
beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
|
||||
2>/dev/null > /tmp/all-tracks.txt
|
||||
|
||||
# Pass artist + album allowlists into awk via -v
|
||||
CROSS_ALBUM_KEEP_RE='^(porterrobinson|madeon|variousartists)$'
|
||||
# Album-name patterns that indicate "intentional alt version" — these rows are
|
||||
# filtered out before grouping, so any group needing 2+ matches will only form
|
||||
# from items whose albums DON'T look like remix/live/single-version releases.
|
||||
# Tagging convention: when the user has multiple legitimate versions of a
|
||||
# track, the alt-version copies live on albums with explicit version markers
|
||||
# in the album title ("(Extended Mix) - Single", "(Remixed)", "Live at X").
|
||||
# Without this filter, Pass 4 would try to dedupe Qrion's "Proud" off the
|
||||
# "I Hope It Lasts Forever (Remixed)" album, Fred again._ studio tracks
|
||||
# against "Apple Music Live" recordings, etc.
|
||||
ALBUM_KEEP_RE='(soundtrack|originalmotionpicture|ost|vgm|gameost|liveat|livein|livefrom|livesession|applemusiclive|applelive|bbclive|bbcsession|mtvunplugged|unplugged|concert|inconcert|extended|remix|remixed|remixes|originalmix|radioedit|radiomix|clubmix|djedit|djmix|dubmix|vocalmix|instrumentalmix|longversion|shortversion|albumversion|originalversion)'
|
||||
|
||||
awk -F"${SEP}" -v sep="$SEP" \
|
||||
-v artist_keep="$CROSS_ALBUM_KEEP_RE" \
|
||||
-v album_keep="$ALBUM_KEEP_RE" '
|
||||
function norm(s, t) {
|
||||
t = tolower(s)
|
||||
gsub(/[^a-z0-9]/, "", t)
|
||||
return t
|
||||
}
|
||||
# Strip trailing mix-version suffix from title so "Foo" and "Foo (Extended Mix)"
|
||||
# share the same base. Only strips generic version markers — does NOT strip
|
||||
# remixer-named suffixes like "(Joris Voorn Remix)", which are different songs.
|
||||
function is_sentinel_title(t, lt) {
|
||||
lt = tolower(t)
|
||||
gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
|
||||
if (lt == "") return 1
|
||||
if (lt == "[unknown]" || lt == "unknown") return 1
|
||||
if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
|
||||
return 0
|
||||
}
|
||||
function strip_mix_suffix(s, t, sfx) {
|
||||
t = tolower(s)
|
||||
sfx = "(extended[[:space:]]*mix|extended[[:space:]]*version|extended|" \
|
||||
"radio[[:space:]]*edit|radio[[:space:]]*mix|radio[[:space:]]*version|" \
|
||||
"original[[:space:]]*mix|original[[:space:]]*version|" \
|
||||
"club[[:space:]]*mix|dj[[:space:]]*edit|dj[[:space:]]*mix|" \
|
||||
"long[[:space:]]*version|short[[:space:]]*version|" \
|
||||
"single[[:space:]]*version|album[[:space:]]*version)"
|
||||
# parenthetical: " (Extended Mix)"
|
||||
t = gensub("[[:space:]]*\\(" sfx "\\)[[:space:]]*$", "", "g", t)
|
||||
# dash-separated: " - Extended Mix"
|
||||
t = gensub("[[:space:]]*-[[:space:]]+" sfx "[[:space:]]*$", "", "g", t)
|
||||
# bare trailing: " Extended Mix"
|
||||
t = gensub("[[:space:]]+" sfx "[[:space:]]*$", "", "g", t)
|
||||
return t
|
||||
}
|
||||
{
|
||||
id = $1; artist = $2; album = $3; title = $4; path = $5
|
||||
if (is_sentinel_title(title)) next
|
||||
na = norm(artist)
|
||||
if (na ~ artist_keep) next
|
||||
if (norm(album) ~ album_keep) next
|
||||
tb = strip_mix_suffix(title)
|
||||
ntb = norm(tb)
|
||||
if (length(ntb) < 4) next
|
||||
k = na sep ntb
|
||||
paths[k] = paths[k] id sep path "\n"
|
||||
count[k]++
|
||||
raws3[k][na sep norm(album) sep norm(title)] = 1
|
||||
albums[k][album] = 1
|
||||
}
|
||||
END {
|
||||
for (k in paths) {
|
||||
if (count[k] < 2) continue
|
||||
# Skip if all items share the same Pass 3 normalized key (Pass 3 already handled).
|
||||
rcount = 0; for (r in raws3[k]) rcount++
|
||||
if (rcount < 2) continue
|
||||
# Require ≥2 distinct album names — otherwise Pass 3 would have grouped them.
|
||||
acount = 0; for (a in albums[k]) acount++
|
||||
if (acount < 2) continue
|
||||
print "===" k "\n" paths[k] "---"
|
||||
}
|
||||
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt
|
||||
|
||||
GROUP_KEY=""
|
||||
GROUP_PATHS=()
|
||||
|
||||
flush_p4_group() {
|
||||
[[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
|
||||
process_group "P4 cross-album: $GROUP_KEY" "${GROUP_PATHS[@]}"
|
||||
GROUP_PATHS=()
|
||||
}
|
||||
|
||||
while IFS= read -r line; do
|
||||
if [[ "$line" == ===* ]]; then
|
||||
flush_p4_group
|
||||
GROUP_KEY="${line#===}"
|
||||
GROUP_PATHS=()
|
||||
elif [[ "$line" == "---" ]]; then
|
||||
flush_p4_group
|
||||
elif [[ -n "$line" ]]; then
|
||||
GROUP_PATHS+=("$line")
|
||||
fi
|
||||
done < /tmp/dups-grouped.txt
|
||||
flush_p4_group # flush last group
|
||||
|
||||
P4_GROUPS=$(grep -c '^--- P4 ' "$LOG" 2>/dev/null; true)
|
||||
log ""
|
||||
log "Pass 4 complete: $P4_GROUPS cross-album dup groups"
|
||||
|
||||
rm -f /tmp/beets-all-paths.txt /tmp/beets-id-paths.txt /tmp/all-tracks.txt /tmp/dups-grouped.txt
|
||||
|
||||
TOTAL_GROUPS=$(( P1_GROUPS + P2_GROUPS + P3_GROUPS + P4_GROUPS ))
|
||||
KEPT=$(grep -c '^ KEEP ' "$LOG" 2>/dev/null; true)
|
||||
DELETED=$(grep -c '^ DELETE ' "$LOG" 2>/dev/null; true)
|
||||
|
||||
log ""
|
||||
log "=== Summary: $TOTAL_GROUPS dup groups ($P1_GROUPS numbered-sibling, $P2_GROUPS case-insensitive, $P3_GROUPS normalized, $P4_GROUPS cross-album), $KEPT kept, $DELETED to delete ==="
|
||||
[[ $APPLY -eq 0 ]] && log "DRY RUN — re-run with --apply to actually delete"
|
||||
Executable
+510
@@ -0,0 +1,510 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
enrich-buy-url.py — best-effort buy links for non-purchase (Soulseek) tracks.
|
||||
|
||||
For every library FLAC that has no buy link yet, look the track up by
|
||||
artist + title and, on an EXACT match, stamp the store page into the
|
||||
COMMERCIAL_INFORMATION tag (same field the Bandcamp-purchase pipeline uses,
|
||||
which AzuraCast maps to the buy_url custom field — see README "Buy links").
|
||||
|
||||
Source cascade (configurable), ordered by quality / DRM-freedom / artist
|
||||
payout: Bandcamp first (lossless, best payout), then Qobuz (DRM-free hi-res),
|
||||
then the iTunes Search API as the last-resort fallback. EXACT matching only —
|
||||
exact artist (primary artist; collabs allowed as a leading match) AND exact
|
||||
title — to keep wrong links out of the files, per the err-toward-false-negatives
|
||||
policy.
|
||||
|
||||
Qobuz needs a logged-in session: it reads a user_auth_token (from --qobuz-token,
|
||||
$QOBUZ_AUTH_TOKEN, or $ALEMBIC_CONFIG_DIR/pipeline/qobuz/token — grab it from the Web Player's
|
||||
X-User-Auth-Token request header) and an app_id (auto-scraped from the Web Player
|
||||
bundle, or --qobuz-app-id / $QOBUZ_APP_ID / $ALEMBIC_CONFIG_DIR/pipeline/qobuz/app_id). With no
|
||||
token, Qobuz is silently dropped from the cascade and the run falls through to
|
||||
the next source.
|
||||
|
||||
Resumable: files that already carry a buy link are skipped, so a re-run only
|
||||
processes new/untagged tracks. Safe to wire into cron after the nightly
|
||||
playlist runs.
|
||||
|
||||
Stdlib only. Dry-run by default.
|
||||
|
||||
Usage:
|
||||
enrich-buy-url.py [--library DIR] [--apply] [--force]
|
||||
[--sources bandcamp,qobuz,itunes] [--limit N]
|
||||
[--bc-sleep S] [--qb-sleep S] [--it-sleep S]
|
||||
[--qobuz-token TOKEN] [--qobuz-app-id ID]
|
||||
[--azuracast-key ID:SECRET] [--azuracast-base URL] [--station N]
|
||||
"""
|
||||
import sys, os, re, json, time, argparse, subprocess
|
||||
import urllib.request, urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
UA = "Mozilla/5.0 (X11; Linux x86_64) enrich-buy-url/1.0"
|
||||
BUY_URL_TAG = "COMMERCIAL_INFORMATION"
|
||||
ITUNES_SEARCH = "https://itunes.apple.com/search"
|
||||
BC_AUTOCOMPLETE = "https://bandcamp.com/api/bcsearch_public_api/1/autocomplete_elastic"
|
||||
QOBUZ_SEARCH = "https://www.qobuz.com/api.json/0.2/track/search"
|
||||
QOBUZ_ALBUM_GET = "https://www.qobuz.com/api.json/0.2/album/get"
|
||||
QOBUZ_LOGIN_PAGE = "https://play.qobuz.com/login"
|
||||
_ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
|
||||
QOBUZ_TOKEN_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/token"
|
||||
QOBUZ_APPID_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/app_id"
|
||||
QOBUZ_REGION_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/region"
|
||||
QOBUZ_DEFAULT_REGION = "ca-en"
|
||||
|
||||
# Populated once by resolve_qobuz() during startup; None disables the source.
|
||||
# Carries {token, app_id, region}.
|
||||
QOBUZ_CREDS = None
|
||||
|
||||
|
||||
# ---- matching --------------------------------------------------------------
|
||||
def norm(s):
|
||||
s = (s or "").lower()
|
||||
s = re.sub(r"\((feat|ft|featuring|with)\.?[^)]*\)", "", s)
|
||||
s = re.sub(r"\b(feat|ft|featuring)\.?\s.*$", "", s)
|
||||
s = re.sub(r"[^a-z0-9]+", " ", s)
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
|
||||
def artist_ok(query, result):
|
||||
"""Exact, or the queried (primary) artist leads a collab credit."""
|
||||
nq, nr = norm(query), norm(result)
|
||||
if not nq or not nr:
|
||||
return False
|
||||
return nr == nq or nr.startswith(nq + " ") or nq.startswith(nr + " ")
|
||||
|
||||
|
||||
def title_ok(query, result):
|
||||
return norm(query) == norm(result) and norm(query) != ""
|
||||
|
||||
|
||||
def url_source(url):
|
||||
"""Best-effort: which store an already-stored buy link came from. Used by
|
||||
--upgrade-from to spot links worth re-evaluating against higher-priority
|
||||
sources. Unknown/white-label domains are treated as Bandcamp (its custom
|
||||
domains, e.g. music.monstercat.com, are the only ones we can't fingerprint)."""
|
||||
u = (url or "").lower()
|
||||
if "apple.com" in u or "itunes" in u:
|
||||
return "itunes"
|
||||
if "qobuz.com" in u:
|
||||
return "qobuz"
|
||||
return "bandcamp"
|
||||
|
||||
|
||||
# ---- sources ---------------------------------------------------------------
|
||||
def _open(req, timeout):
|
||||
"""One retry with backoff on throttle/temporary errors (polite + better yield)."""
|
||||
for attempt in range(2):
|
||||
try:
|
||||
return urllib.request.urlopen(req, timeout=timeout).read()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (429, 500, 502, 503) and attempt == 0:
|
||||
time.sleep(5)
|
||||
continue
|
||||
raise
|
||||
|
||||
|
||||
def _get(url, timeout=30):
|
||||
return _open(urllib.request.Request(url, headers={"User-Agent": UA}), timeout)
|
||||
|
||||
|
||||
def _post(url, body, timeout=30):
|
||||
req = urllib.request.Request(
|
||||
url, data=json.dumps(body).encode(),
|
||||
headers={"User-Agent": UA, "Content-Type": "application/json"})
|
||||
return _open(req, timeout)
|
||||
|
||||
|
||||
def lookup_bandcamp(artist, title):
|
||||
"""Return a track-page URL on an exact match, else None."""
|
||||
try:
|
||||
raw = _post(BC_AUTOCOMPLETE, {
|
||||
"search_text": f"{artist} {title}",
|
||||
"search_filter": "", "full_page": False, "fan_id": None})
|
||||
results = (json.loads(raw).get("auto") or {}).get("results", [])
|
||||
except Exception:
|
||||
return None
|
||||
for r in results:
|
||||
if r.get("type") != "t": # tracks only (exact title)
|
||||
continue
|
||||
if artist_ok(artist, r.get("band_name", "")) and title_ok(title, r.get("name", "")):
|
||||
return r.get("item_url_path") or r.get("url")
|
||||
return None
|
||||
|
||||
|
||||
def lookup_itunes(artist, title):
|
||||
"""Return a store URL on an exact match (prefer buyable), else None."""
|
||||
try:
|
||||
url = ITUNES_SEARCH + "?" + urllib.parse.urlencode(
|
||||
{"term": f"{artist} {title}", "entity": "song", "limit": 8})
|
||||
results = json.loads(_get(url)).get("results", [])
|
||||
except Exception:
|
||||
return None
|
||||
fallback = None
|
||||
for r in results:
|
||||
if artist_ok(artist, r.get("artistName", "")) and title_ok(title, r.get("trackName", "")):
|
||||
link = r.get("trackViewUrl")
|
||||
if not link:
|
||||
continue
|
||||
if r.get("trackPrice"): # buyable wins
|
||||
return link
|
||||
fallback = fallback or link
|
||||
return fallback
|
||||
|
||||
|
||||
def scrape_qobuz_app_id(log):
|
||||
"""Pull the production app_id out of the Web Player bundle. Best-effort:
|
||||
returns None if the page/bundle layout changed."""
|
||||
try:
|
||||
page = _get(QOBUZ_LOGIN_PAGE, timeout=30).decode("utf-8", "replace")
|
||||
m = re.search(r"/resources/[\w.\-]+/bundle\.js", page)
|
||||
if not m:
|
||||
log(" [qobuz] could not locate bundle.js on the login page")
|
||||
return None
|
||||
bundle = _get("https://play.qobuz.com" + m.group(0), timeout=90).decode("utf-8", "replace")
|
||||
m = re.search(r'production:\{api:\{appId:"(\d+)"', bundle)
|
||||
if not m:
|
||||
log(" [qobuz] could not find appId in bundle.js")
|
||||
return None
|
||||
return m.group(1)
|
||||
except Exception as e:
|
||||
log(f" [qobuz] app_id scrape failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def resolve_qobuz(args, log):
|
||||
"""Resolve {token, app_id, region} for the Qobuz source, or None to disable
|
||||
it. Token: --qobuz-token, $QOBUZ_AUTH_TOKEN, then the keyfile. app_id: flag/
|
||||
env/keyfile, else auto-scraped from the Web Player bundle. region: flag/env/
|
||||
keyfile, else the default (controls which store the buy link points at)."""
|
||||
token = args.qobuz_token or os.environ.get("QOBUZ_AUTH_TOKEN")
|
||||
if not token and os.path.exists(QOBUZ_TOKEN_FILE):
|
||||
token = Path(QOBUZ_TOKEN_FILE).read_text().strip()
|
||||
if not token:
|
||||
log(" [qobuz] no user_auth_token (flag/env/keyfile) — dropping Qobuz from cascade")
|
||||
return None
|
||||
|
||||
app_id = args.qobuz_app_id or os.environ.get("QOBUZ_APP_ID")
|
||||
if not app_id and os.path.exists(QOBUZ_APPID_FILE):
|
||||
app_id = Path(QOBUZ_APPID_FILE).read_text().strip()
|
||||
if not app_id:
|
||||
app_id = scrape_qobuz_app_id(log)
|
||||
if not app_id:
|
||||
log(" [qobuz] no app_id (flag/env/keyfile/scrape) — dropping Qobuz from cascade")
|
||||
return None
|
||||
|
||||
region = getattr(args, "qobuz_region", None) or os.environ.get("QOBUZ_REGION")
|
||||
if not region and os.path.exists(QOBUZ_REGION_FILE):
|
||||
region = Path(QOBUZ_REGION_FILE).read_text().strip()
|
||||
region = (region or QOBUZ_DEFAULT_REGION).strip().strip("/")
|
||||
return {"token": token.strip(), "app_id": str(app_id).strip(), "region": region}
|
||||
|
||||
|
||||
def qobuz_store_url(album_id):
|
||||
"""Resolve an album id to its canonical, regioned *purchase* page
|
||||
(www.qobuz.com/<region>/album/<slug>/<id>) via album/get. Returns None if the
|
||||
album isn't purchasable or can't be fetched — so the caller falls through to
|
||||
the next source rather than stamping a streaming-only link. We deliberately
|
||||
do NOT use open.qobuz.com (that's the player/streaming deep link)."""
|
||||
if not (QOBUZ_CREDS and album_id):
|
||||
return None
|
||||
try:
|
||||
url = QOBUZ_ALBUM_GET + "?" + urllib.parse.urlencode(
|
||||
{"album_id": album_id, "app_id": QOBUZ_CREDS["app_id"]})
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": UA,
|
||||
"X-App-Id": QOBUZ_CREDS["app_id"],
|
||||
"X-User-Auth-Token": QOBUZ_CREDS["token"]})
|
||||
d = json.loads(_open(req, 30))
|
||||
except Exception:
|
||||
return None
|
||||
# album/get returns a region-less relative_url like /album/<slug>/<id>; the
|
||||
# bare `url` is locale-pinned to the account default (fr-fr), so we build from
|
||||
# relative_url + the configured region instead.
|
||||
rel = d.get("relative_url")
|
||||
if not d.get("purchasable") or not rel:
|
||||
return None
|
||||
return f"https://www.qobuz.com/{QOBUZ_CREDS['region']}{rel}"
|
||||
|
||||
|
||||
def lookup_qobuz(artist, title):
|
||||
"""Return a Qobuz *purchase* URL on an exact, purchasable match, else None
|
||||
(falls through to the next source). Requires QOBUZ_CREDS."""
|
||||
if not QOBUZ_CREDS:
|
||||
return None
|
||||
try:
|
||||
url = QOBUZ_SEARCH + "?" + urllib.parse.urlencode(
|
||||
{"query": f"{artist} {title}", "limit": 10, "app_id": QOBUZ_CREDS["app_id"]})
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": UA,
|
||||
"X-App-Id": QOBUZ_CREDS["app_id"],
|
||||
"X-User-Auth-Token": QOBUZ_CREDS["token"]})
|
||||
items = (json.loads(_open(req, 30)).get("tracks") or {}).get("items", [])
|
||||
except Exception:
|
||||
return None
|
||||
tried = 0
|
||||
for t in items:
|
||||
album = t.get("album") or {}
|
||||
performer = (t.get("performer") or {}).get("name", "") or \
|
||||
(album.get("artist") or {}).get("name", "")
|
||||
if not (artist_ok(artist, performer) and title_ok(title, t.get("title", ""))):
|
||||
continue
|
||||
# Search carries only the album id; album/get yields the purchase page and
|
||||
# gates on purchasability. A given track can appear on several releases
|
||||
# (single, album, comp) — try the first few for a purchasable one before
|
||||
# giving up and falling through to iTunes.
|
||||
time.sleep(0.3) # polite: a second call per match
|
||||
link = qobuz_store_url(album.get("id"))
|
||||
if link:
|
||||
return link
|
||||
tried += 1
|
||||
if tried >= 3:
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
SOURCES = {"bandcamp": lookup_bandcamp, "qobuz": lookup_qobuz, "itunes": lookup_itunes}
|
||||
|
||||
|
||||
# ---- tag I/O ---------------------------------------------------------------
|
||||
def flac_tag(path, name):
|
||||
r = subprocess.run(["metaflac", f"--show-tag={name}", path], capture_output=True, text=True)
|
||||
for line in r.stdout.splitlines():
|
||||
if "=" in line:
|
||||
return line.split("=", 1)[1]
|
||||
return ""
|
||||
|
||||
|
||||
def set_flac_tag(path, name, value):
|
||||
subprocess.run(["metaflac", f"--remove-tag={name}", path], capture_output=True)
|
||||
return subprocess.run(["metaflac", f"--set-tag={name}={value}", path],
|
||||
capture_output=True).returncode == 0
|
||||
|
||||
|
||||
# ---- AzuraCast reprocess (same as backfill-buy-url.py) ----------------------
|
||||
def az_reprocess(base, key, station, host_paths, library, log):
|
||||
root = library.rstrip("/") + "/"
|
||||
want = {p[len(root):] for p in host_paths if p.startswith(root)}
|
||||
headers = {"X-API-Key": key, "Accept": "application/json"}
|
||||
|
||||
def get(p):
|
||||
return json.loads(urllib.request.urlopen(
|
||||
urllib.request.Request(base + p, headers=headers), timeout=90).read())
|
||||
|
||||
uids, page = [], 1
|
||||
while True:
|
||||
d = get(f"/api/station/{station}/files?per_page=500&page={page}")
|
||||
for row in d["rows"]:
|
||||
if row["path"] in want:
|
||||
uids.append(row["unique_id"])
|
||||
if page >= d["total_pages"]:
|
||||
break
|
||||
page += 1
|
||||
if not uids:
|
||||
log(" no matching AzuraCast files to reprocess")
|
||||
return
|
||||
# AzuraCast caps batch size in practice; chunk to be safe.
|
||||
for i in range(0, len(uids), 200):
|
||||
chunk = uids[i:i + 200]
|
||||
req = urllib.request.Request(
|
||||
base + f"/api/station/{station}/files/batch",
|
||||
data=json.dumps({"do": "reprocess", "files": chunk}).encode(), method="PUT",
|
||||
headers={**headers, "Content-Type": "application/json"})
|
||||
urllib.request.urlopen(req, timeout=120).read()
|
||||
log(f" queued reprocess for {len(uids)} AzuraCast file(s)")
|
||||
|
||||
|
||||
def refresh_qobuz_links(flacs, args, az_key):
|
||||
"""Rewrite existing open.qobuz.com (streaming) buy links to the canonical
|
||||
www.qobuz.com purchase page, converting by album id (no re-matching). Links
|
||||
that aren't purchasable / can't be resolved are left intact and counted."""
|
||||
print(f"[refresh-qobuz] mode={'APPLY' if args.apply else 'DRY RUN'} "
|
||||
f"region={QOBUZ_CREDS['region']}")
|
||||
print(f"[refresh-qobuz] scanning {len(flacs)} FLAC files...\n")
|
||||
looked = converted = unconvertible = 0
|
||||
touched = []
|
||||
for p in flacs:
|
||||
sp = str(p)
|
||||
existing = flac_tag(sp, BUY_URL_TAG)
|
||||
if not existing or "open.qobuz.com" not in existing:
|
||||
continue
|
||||
m = re.search(r"open\.qobuz\.com/album/([^/?#]+)", existing)
|
||||
if not m:
|
||||
continue # track-form deep link: leave it
|
||||
if args.limit and looked >= args.limit:
|
||||
break
|
||||
looked += 1
|
||||
time.sleep(args.qb_sleep)
|
||||
new = qobuz_store_url(m.group(1))
|
||||
rel = sp[len(args.library):].lstrip("/")
|
||||
if not new or new == existing:
|
||||
unconvertible += 1 # not purchasable in-region / gone
|
||||
continue
|
||||
converted += 1
|
||||
print(f" ~ {rel}\n {existing}\n -> {new}")
|
||||
if args.apply:
|
||||
if set_flac_tag(sp, BUY_URL_TAG, new):
|
||||
touched.append(sp)
|
||||
else:
|
||||
print(" ! metaflac write failed")
|
||||
else:
|
||||
touched.append(sp)
|
||||
|
||||
print(f"\n[refresh-qobuz] {looked} open.qobuz links, "
|
||||
f"{converted} {'converted' if args.apply else 'would-convert'}, "
|
||||
f"{unconvertible} left intact (not purchasable / unresolved)")
|
||||
|
||||
if args.apply and touched and az_key:
|
||||
print("[refresh-qobuz] telling AzuraCast to reprocess touched files...")
|
||||
az_reprocess(args.azuracast_base, az_key, args.station,
|
||||
touched, args.library, lambda m: print(m))
|
||||
elif not args.apply:
|
||||
print("[refresh-qobuz] re-run with --apply to write")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--library", default=f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library")
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--force", action="store_true", help="re-match files that already have a buy link")
|
||||
ap.add_argument("--upgrade-from", default="",
|
||||
help="comma-separated sources whose existing links should be "
|
||||
"re-evaluated and REPLACED if a higher-priority source now "
|
||||
"matches (e.g. 'itunes'). Lower-priority matches never "
|
||||
"overwrite; a no-match leaves the existing link intact.")
|
||||
ap.add_argument("--sources", default="bandcamp,qobuz,itunes",
|
||||
help="comma-separated cascade order (subset of: bandcamp, qobuz, itunes)")
|
||||
ap.add_argument("--limit", type=int, default=0, help="stop after N lookups (0 = all)")
|
||||
ap.add_argument("--bc-sleep", type=float, default=0.7)
|
||||
ap.add_argument("--qb-sleep", type=float, default=0.8)
|
||||
ap.add_argument("--it-sleep", type=float, default=1.0)
|
||||
ap.add_argument("--qobuz-token", default=None,
|
||||
help="Qobuz user_auth_token (else $QOBUZ_AUTH_TOKEN or "
|
||||
f"{QOBUZ_TOKEN_FILE})")
|
||||
ap.add_argument("--qobuz-app-id", default=None,
|
||||
help="Qobuz app_id (else $QOBUZ_APP_ID, keyfile, or auto-scrape)")
|
||||
ap.add_argument("--qobuz-region", default=None,
|
||||
help=f"Qobuz store region for buy links, e.g. ca-en (else "
|
||||
f"$QOBUZ_REGION, {QOBUZ_REGION_FILE}, or {QOBUZ_DEFAULT_REGION})")
|
||||
ap.add_argument("--refresh-qobuz", action="store_true",
|
||||
help="one-shot: rewrite existing open.qobuz.com (streaming) buy "
|
||||
"links to the canonical www.qobuz.com purchase page. No "
|
||||
"re-matching — converts by album id. Skips links that "
|
||||
"aren't purchasable (leaves them intact).")
|
||||
ap.add_argument("--azuracast-key", default=None)
|
||||
ap.add_argument("--azuracast-base", default="https://admin.ephemeral.club")
|
||||
ap.add_argument("--station", type=int, default=1)
|
||||
args = ap.parse_args()
|
||||
|
||||
# Resolve the AzuraCast key (for the reprocess trigger) without putting the
|
||||
# secret on the command line / in cron: --azuracast-key, then env, then file.
|
||||
az_key = args.azuracast_key or os.environ.get("AZURACAST_API_KEY")
|
||||
if not az_key:
|
||||
keyfile = f"{_ALEMBIC_CONFIG_DIR}/pipeline/azuracast/api_key"
|
||||
if os.path.exists(keyfile):
|
||||
az_key = Path(keyfile).read_text().strip()
|
||||
|
||||
cascade = [s.strip() for s in args.sources.split(",") if s.strip() in SOURCES]
|
||||
if not cascade:
|
||||
sys.exit(f"[enrich] no valid sources in {args.sources!r}")
|
||||
sleep_for = {"bandcamp": args.bc_sleep, "qobuz": args.qb_sleep, "itunes": args.it_sleep}
|
||||
|
||||
# Qobuz needs auth; resolve it now and drop the source if unavailable so the
|
||||
# cascade just falls through (err-toward-false-negatives — no guessed links).
|
||||
# --refresh-qobuz also needs the creds (to call album/get).
|
||||
global QOBUZ_CREDS
|
||||
if "qobuz" in cascade or args.refresh_qobuz:
|
||||
QOBUZ_CREDS = resolve_qobuz(args, lambda m: print(m))
|
||||
if not QOBUZ_CREDS:
|
||||
cascade = [s for s in cascade if s != "qobuz"]
|
||||
if args.refresh_qobuz:
|
||||
sys.exit("[enrich] --refresh-qobuz needs Qobuz creds — aborting")
|
||||
|
||||
upgrade_from = {s.strip() for s in args.upgrade_from.split(",")
|
||||
if s.strip() in SOURCES}
|
||||
|
||||
flacs = list(Path(args.library).rglob("*.flac"))
|
||||
|
||||
# --- one-shot migration: open.qobuz.com (streaming) -> purchase page -------
|
||||
if args.refresh_qobuz:
|
||||
refresh_qobuz_links(flacs, args, az_key)
|
||||
return
|
||||
|
||||
print(f"[enrich] mode={'APPLY' if args.apply else 'DRY RUN'} cascade={cascade}"
|
||||
+ (f" upgrade-from={sorted(upgrade_from)}" if upgrade_from else ""))
|
||||
print(f"[enrich] {len(flacs)} FLAC files in library\n")
|
||||
|
||||
looked = found = upgraded = 0
|
||||
by_source = {s: 0 for s in cascade}
|
||||
touched = []
|
||||
for p in flacs:
|
||||
sp = str(p)
|
||||
existing = flac_tag(sp, BUY_URL_TAG)
|
||||
# Decide which sources to actually query for this file.
|
||||
if existing and not args.force:
|
||||
cur = url_source(existing)
|
||||
if upgrade_from and cur in upgrade_from:
|
||||
# Only try sources ranked ABOVE the current one — a same/lower
|
||||
# match must never replace it. No-match leaves the link intact.
|
||||
run_sources = cascade[:cascade.index(cur)] if cur in cascade else list(cascade)
|
||||
if not run_sources:
|
||||
continue
|
||||
else:
|
||||
continue # already tagged, not up for upgrade
|
||||
else:
|
||||
run_sources = cascade # untagged, or --force
|
||||
artist = flac_tag(sp, "ARTIST").split(";")[0].strip() or flac_tag(sp, "ALBUMARTIST")
|
||||
title = flac_tag(sp, "TITLE")
|
||||
if not (artist and title):
|
||||
continue
|
||||
if args.limit and looked >= args.limit:
|
||||
break
|
||||
looked += 1
|
||||
|
||||
url = src = None
|
||||
for s in run_sources:
|
||||
url = SOURCES[s](artist, title)
|
||||
time.sleep(sleep_for[s])
|
||||
if url:
|
||||
src = s
|
||||
break
|
||||
|
||||
rel = sp[len(args.library):].lstrip("/")
|
||||
if not url:
|
||||
continue
|
||||
found += 1
|
||||
by_source[src] += 1
|
||||
if existing:
|
||||
upgraded += 1
|
||||
print(f" ~ [{src}] {artist} - {title}\n -> {url}\n (replaces {existing})")
|
||||
else:
|
||||
print(f" + [{src}] {artist} - {title}\n -> {url}")
|
||||
if args.apply:
|
||||
if set_flac_tag(sp, BUY_URL_TAG, url):
|
||||
touched.append(sp)
|
||||
else:
|
||||
print(" ! metaflac write failed")
|
||||
else:
|
||||
touched.append(sp)
|
||||
|
||||
if found % 50 == 0:
|
||||
print(f" ...{found} links from {looked} lookups so far", flush=True)
|
||||
|
||||
print(f"\n[enrich] {looked} lookups, {found} {'tagged' if args.apply else 'would-tag'}"
|
||||
f" ({', '.join(f'{s}={by_source[s]}' for s in cascade)})"
|
||||
f" no-match={looked - found}"
|
||||
+ (f" upgraded={upgraded}" if upgrade_from else ""))
|
||||
|
||||
if args.apply and touched and az_key:
|
||||
print("[enrich] telling AzuraCast to reprocess touched files...")
|
||||
az_reprocess(args.azuracast_base, az_key, args.station,
|
||||
touched, args.library, lambda m: print(m))
|
||||
elif args.apply and touched and not az_key:
|
||||
print("[enrich] no AzuraCast key (flag/env/keyfile) — tags written; "
|
||||
"AzuraCast's periodic media scan will pick them up.")
|
||||
elif not args.apply:
|
||||
print("[enrich] re-run with --apply to write (key via --azuracast-key, "
|
||||
"$AZURACAST_API_KEY, or $ALEMBIC_CONFIG_DIR/pipeline/azuracast/api_key to reprocess)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+310
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
export-laptop-playlists.py — daily Navidrome → Traktor M3U export.
|
||||
|
||||
Lists every Navidrome playlist whose name starts with `dj-`, fetches each
|
||||
playlist's tracks via Subsonic, looks up the real on-disk path of each
|
||||
track in beets, and writes a `.m3u` for Syncthing → Traktor.
|
||||
|
||||
Output dir: $MUSIC_DATA_DIR/playlists-laptop/
|
||||
- One `dj-<name>.m3u` per Navidrome dj-* playlist (overwritten each run)
|
||||
- Exact format Traktor Pro 4 itself uses when you Export Playlist as M3U:
|
||||
one absolute Windows path per line (`C:\\Music\\Library\\...\\Track.flac`),
|
||||
UTF-8 encoded, LF line endings, NO `#EXTM3U` header. Traktor's M3U
|
||||
importer treats `#EXTM3U` as either a parse error or a phantom track
|
||||
path, which kicks the rest of the import into "treat-as-new" mode and
|
||||
the playlist ends up with unanalyzed records. With the header omitted,
|
||||
Traktor parses each line as a LOCATION, matches against the existing
|
||||
Collection by DIR+FILE+VOLUME, and the playlist inherits BPM, waveform,
|
||||
and key from the already-analyzed entries.
|
||||
|
||||
Why beets, not Subsonic, for paths: Navidrome's Subsonic `path` attribute
|
||||
is a *synthesized* string like `Artist/Album/01 - Title.flac` even when
|
||||
the real file is just `Artist/Album/Title.flac`. The track-number prefix
|
||||
is generated from metadata, not read from disk. Traktor then can't find
|
||||
the file. Beets stores the real path, so we look up each track by
|
||||
(album, title) in a one-shot beets index, falling back to artist
|
||||
disambiguation when multiple albums share the same album+title.
|
||||
|
||||
Reconciles deletes: any `dj-*.m3u` on disk whose stem is no longer a
|
||||
current Navidrome playlist is removed; also sweeps legacy `.m3u8`/`.nml`
|
||||
files from earlier experiments.
|
||||
|
||||
Existing M3U files in `$MUSIC_DATA_DIR/playlists/` (Navidrome's M3U
|
||||
import source for the automated Spotify pipeline) are NOT touched.
|
||||
|
||||
Usage:
|
||||
export-laptop-playlists.py [--verbose]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
_ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
|
||||
_MUSIC_DATA_DIR = os.environ.get("MUSIC_DATA_DIR", "/data/music")
|
||||
|
||||
OUT_DIR = f"{_MUSIC_DATA_DIR}/playlists-laptop"
|
||||
|
||||
# Absolute Windows path to the laptop's library root.
|
||||
LIBRARY_LAPTOP_ROOT = r"C:\Music\Library"
|
||||
|
||||
# Beets stores paths with this prefix (the container-side mount of the
|
||||
# library); strip it to get the path relative to the library root. This is
|
||||
# still "/music/" through the migration's Stage 0-3 transitional mount;
|
||||
# revisit at Stage 4 once beets' directory: becomes MUSIC_DATA_DIR/Library.
|
||||
BEETS_LIBRARY_PREFIX = "/music/"
|
||||
|
||||
M3U_EXT = ".m3u"
|
||||
# Older formats from earlier iterations of this script — swept by reconcile.
|
||||
LEGACY_EXTS = (".m3u8", ".nml")
|
||||
|
||||
|
||||
def _load_navidrome_env():
|
||||
path = f"{_ALEMBIC_CONFIG_DIR}/pipeline/navidrome/admin.env"
|
||||
env = {}
|
||||
if os.path.exists(path):
|
||||
for line in open(path):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env[k] = v.strip().strip("'").strip('"')
|
||||
return env
|
||||
|
||||
_nd_env = _load_navidrome_env()
|
||||
ND_BASE = _nd_env.get("ND_BASE", "http://navidrome:4533")
|
||||
ND_USER = _nd_env.get("ND_USER", "andrew")
|
||||
ND_PASS = _nd_env.get("ND_PASS", "")
|
||||
ND_API_VERSION = "1.16.0"
|
||||
ND_CLIENT = "dj-export"
|
||||
|
||||
NS = "{http://subsonic.org/restapi}"
|
||||
|
||||
# ASCII Unit Separator — safe field separator for beets `-f` output.
|
||||
SEP = "\x1f"
|
||||
|
||||
|
||||
def _call(endpoint: str, params: dict[str, str]) -> ET.Element:
|
||||
qp = {
|
||||
"u": ND_USER, "p": ND_PASS,
|
||||
"v": ND_API_VERSION, "c": ND_CLIENT, "f": "xml",
|
||||
**params,
|
||||
}
|
||||
url = f"{ND_BASE}/rest/{endpoint}.view?{urllib.parse.urlencode(qp)}"
|
||||
with urllib.request.urlopen(url, timeout=60) as r:
|
||||
body = r.read()
|
||||
root = ET.fromstring(body)
|
||||
if root.attrib.get("status") != "ok":
|
||||
err = root.find(f"{NS}error")
|
||||
msg = err.attrib.get("message", "unknown") if err is not None else "?"
|
||||
raise RuntimeError(f"Subsonic {endpoint} failed: {msg}")
|
||||
return root
|
||||
|
||||
|
||||
def list_dj_playlists() -> list[tuple[str, str]]:
|
||||
root = _call("getPlaylists", {})
|
||||
out: list[tuple[str, str]] = []
|
||||
for p in root.iter(f"{NS}playlist"):
|
||||
name = p.attrib.get("name", "")
|
||||
if name.startswith("dj-"):
|
||||
out.append((p.attrib["id"], name))
|
||||
return out
|
||||
|
||||
|
||||
def get_playlist_tracks(pid: str) -> list[tuple[str, str, str]]:
|
||||
"""Return (albumartist, album, title) tuples per playlist entry."""
|
||||
root = _call("getPlaylist", {"id": pid})
|
||||
out: list[tuple[str, str, str]] = []
|
||||
for entry in root.iter(f"{NS}entry"):
|
||||
aa = (entry.attrib.get("displayAlbumArtist")
|
||||
or entry.attrib.get("artist", ""))
|
||||
album = entry.attrib.get("album", "")
|
||||
title = entry.attrib.get("title", "")
|
||||
out.append((aa, album, title))
|
||||
return out
|
||||
|
||||
|
||||
def _normkey(s: str) -> str:
|
||||
"""Lowercase + normalize curly quotes to straight ASCII."""
|
||||
return (s.lower()
|
||||
.replace("’", "'").replace("‘", "'")
|
||||
.replace("“", '"').replace("”", '"'))
|
||||
|
||||
|
||||
def build_beets_index() -> dict[tuple[str, str], list[tuple[str, str, str, str]]]:
|
||||
"""Map (album, title) →
|
||||
[(albumartist, album, title, library-relative path), ...].
|
||||
|
||||
Album and title are returned with original casing (not _normkey'd) so the
|
||||
NML COLLECTION entries can carry the real metadata for display."""
|
||||
fmt = SEP.join(["$albumartist", "$album", "$title", "$path"])
|
||||
res = subprocess.run(
|
||||
["beet", "ls", "-f", fmt],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
idx: dict[tuple[str, str], list[tuple[str, str, str, str]]] = {}
|
||||
for ln in res.stdout.splitlines():
|
||||
parts = ln.split(SEP)
|
||||
if len(parts) != 4:
|
||||
continue
|
||||
aa, alb, ti, path = parts
|
||||
if not path.startswith(BEETS_LIBRARY_PREFIX):
|
||||
continue
|
||||
rel = path[len(BEETS_LIBRARY_PREFIX):]
|
||||
idx.setdefault((_normkey(alb), _normkey(ti)), []).append((aa, alb, ti, rel))
|
||||
return idx
|
||||
|
||||
|
||||
def resolve(idx: dict[tuple[str, str], list[tuple[str, str, str, str]]],
|
||||
aa_query: str, album: str, title: str
|
||||
) -> tuple[str, str, str, str] | None:
|
||||
"""Return (albumartist, album, title, rel_path) for a Subsonic track, or None."""
|
||||
candidates = idx.get((_normkey(album), _normkey(title)))
|
||||
if not candidates:
|
||||
return None
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
q = _normkey(aa_query).replace(" • ", " ").replace(";", " ")
|
||||
for c in candidates:
|
||||
if _normkey(c[0]) in q:
|
||||
return c
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def to_windows_path(rel: str) -> str:
|
||||
"""Library-relative POSIX path → absolute Windows path with backslashes."""
|
||||
return LIBRARY_LAPTOP_ROOT + "\\" + rel.replace("/", "\\")
|
||||
|
||||
|
||||
def write_m3u(name: str, tracks: list[tuple[str, str, str, str]], verbose: bool) -> int:
|
||||
"""Plain-path M3U, no `#EXTM3U` header — matches the format Traktor
|
||||
Pro 4 itself emits via Export Playlist → M3U. With this byte-exact
|
||||
format, Traktor's import re-uses existing analyzed Collection entries
|
||||
(matched by DIR+FILE+VOLUME) instead of creating fresh unanalyzed
|
||||
records."""
|
||||
target = os.path.join(OUT_DIR, f"{name}{M3U_EXT}")
|
||||
tmp = target + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8", newline="\n") as f:
|
||||
for _aa, _alb, _ti, rel in tracks:
|
||||
f.write(to_windows_path(rel) + "\n")
|
||||
os.replace(tmp, target)
|
||||
if verbose:
|
||||
print(f" wrote {target} ({len(tracks)} tracks)")
|
||||
return len(tracks)
|
||||
|
||||
|
||||
def reconcile(current_names: set[str], verbose: bool) -> int:
|
||||
"""Delete dj-*.m3u files whose stem is no longer current, and sweep
|
||||
any legacy .m3u8/.nml leftovers from earlier iterations."""
|
||||
removed = 0
|
||||
if not os.path.isdir(OUT_DIR):
|
||||
return 0
|
||||
for entry in os.scandir(OUT_DIR):
|
||||
if not entry.is_file() or not entry.name.startswith("dj-"):
|
||||
continue
|
||||
if entry.name.endswith(M3U_EXT):
|
||||
stem = entry.name[:-len(M3U_EXT)]
|
||||
if stem in current_names:
|
||||
continue
|
||||
elif any(entry.name.endswith(e) for e in LEGACY_EXTS):
|
||||
pass # always sweep
|
||||
else:
|
||||
continue
|
||||
os.remove(entry.path)
|
||||
removed += 1
|
||||
if verbose:
|
||||
print(f" removed stale {entry.path}")
|
||||
return removed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--verbose", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
stamp = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
print(f"[{stamp}] export-laptop-playlists starting")
|
||||
|
||||
idx = build_beets_index()
|
||||
print(f" beets index: {len(idx)} (album,title) keys")
|
||||
|
||||
playlists = list_dj_playlists()
|
||||
print(f" found {len(playlists)} dj-* playlists in Navidrome")
|
||||
|
||||
# Safety guard: if Navidrome answers successfully but reports ZERO dj-*
|
||||
# playlists while exported files exist on disk, something is wrong on the
|
||||
# Navidrome side (fresh/restored DB, purge, renamed playlists). Running
|
||||
# reconcile() would delete every dj-*.m3u and Syncthing would propagate
|
||||
# the deletions to the laptop — the 2026-06-01 Traktor wipe through the
|
||||
# other door. Refuse and keep the files.
|
||||
if not playlists:
|
||||
existing = [e.name for e in os.scandir(OUT_DIR)
|
||||
if e.is_file() and e.name.startswith("dj-")]
|
||||
if existing:
|
||||
print(f" WARN: Navidrome returned 0 dj-* playlists but "
|
||||
f"{len(existing)} dj-* file(s) exist in {OUT_DIR} — "
|
||||
f"REFUSING to reconcile/delete. Check Navidrome before "
|
||||
f"trusting this.")
|
||||
return 0
|
||||
|
||||
total_tracks = 0
|
||||
total_unresolved = 0
|
||||
written = 0
|
||||
skipped = 0
|
||||
for pid, name in playlists:
|
||||
try:
|
||||
tracks = get_playlist_tracks(pid)
|
||||
except Exception as e:
|
||||
print(f" WARN: failed to fetch {name}: {e}")
|
||||
continue
|
||||
resolved: list[tuple[str, str, str, str]] = []
|
||||
unresolved: list[tuple[str, str, str]] = []
|
||||
for aa, album, title in tracks:
|
||||
hit = resolve(idx, aa, album, title)
|
||||
if hit is None:
|
||||
unresolved.append((aa, album, title))
|
||||
continue
|
||||
resolved.append(hit)
|
||||
if unresolved:
|
||||
total_unresolved += len(unresolved)
|
||||
print(f" WARN: {name}: {len(unresolved)} track(s) not in beets index "
|
||||
f"(skipped). First: {unresolved[0]}")
|
||||
# Safety guard: never blow away a populated .m3u with an empty export.
|
||||
# On 2026-06-01 a PURGEMISSING scan emptied the Navidrome dj-* playlists
|
||||
# while the QNAP share was flaky; this script then overwrote every
|
||||
# laptop .m3u with 0 tracks, and the laptop's auto-merge wiped Traktor.
|
||||
# If Navidrome resolves to 0 tracks but a non-empty file already exists,
|
||||
# keep the file and warn loudly instead of trusting the empty result.
|
||||
if not resolved:
|
||||
target = os.path.join(OUT_DIR, f"{name}{M3U_EXT}")
|
||||
if os.path.exists(target) and os.path.getsize(target) > 0:
|
||||
print(f" WARN: {name}: Navidrome returned 0 resolvable tracks but "
|
||||
f"{target} is non-empty — REFUSING to overwrite (kept existing). "
|
||||
f"Check Navidrome (likely a purge/scan glitch) before trusting this.")
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
total_tracks += write_m3u(name, resolved, args.verbose)
|
||||
written += 1
|
||||
|
||||
current_names = {name for _pid, name in playlists}
|
||||
removed = reconcile(current_names, args.verbose)
|
||||
|
||||
print(f"[{stamp}] done. wrote {written} playlists, {total_tracks} total tracks, "
|
||||
f"{total_unresolved} unresolved, {skipped} kept (empty-guard), "
|
||||
f"removed {removed} stale files")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+287
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
find-fuzzy-dupes.py — find acoustic duplicates in the beets library using
|
||||
the existing $ALEMBIC_CONFIG_DIR/beets/fingerprints.db (Chromaprint).
|
||||
|
||||
Pairs whose duration differs by <=5s and Chromaprint similarity >= 0.92 are
|
||||
flagged as duplicates. Groups are transitively merged. Within each group the
|
||||
ranking is: FLAC > MP3/other, then largest size wins.
|
||||
|
||||
DRY RUN by default. Pass --apply to delete losers via `beet remove -d -f`.
|
||||
|
||||
Usage:
|
||||
find-fuzzy-dupes.py
|
||||
find-fuzzy-dupes.py --apply
|
||||
find-fuzzy-dupes.py --threshold 0.95 --apply
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
import numpy as np
|
||||
|
||||
FINGERPRINT_DB = f"{os.environ.get('ALEMBIC_CONFIG_DIR', '/config')}/beets/fingerprints.db"
|
||||
SCAN_CACHE = "/tmp/find-fuzzy-dupes-scan-cache.json"
|
||||
DEFAULT_THRESHOLD = 0.92
|
||||
DURATION_TOLERANCE = 5 # seconds
|
||||
# Library dedup compares same-track recordings — offsets are tiny. Use a
|
||||
# smaller window than DJ-import (±80) so a 4k-file library scans in minutes.
|
||||
MAX_OFFSET = 15
|
||||
|
||||
|
||||
# Precomputed bitmasks for popcount (reused across compares).
|
||||
_M1 = np.uint32(0x55555555)
|
||||
_M2 = np.uint32(0x33333333)
|
||||
_M4 = np.uint32(0x0F0F0F0F)
|
||||
_H01 = np.uint32(0x01010101)
|
||||
|
||||
|
||||
def _sim_at_offset(fp1: np.ndarray, fp2: np.ndarray, offset: int) -> float:
|
||||
if offset >= 0:
|
||||
n = min(fp1.size - offset, fp2.size)
|
||||
if n < 100:
|
||||
return 0.0
|
||||
xor = np.bitwise_xor(fp1[offset : offset + n], fp2[:n])
|
||||
else:
|
||||
n = min(fp1.size, fp2.size + offset)
|
||||
if n < 100:
|
||||
return 0.0
|
||||
xor = np.bitwise_xor(fp1[:n], fp2[-offset : -offset + n])
|
||||
v = xor - ((xor >> 1) & _M1)
|
||||
v = (v & _M2) + ((v >> 2) & _M2)
|
||||
v = (v + (v >> 4)) & _M4
|
||||
diff_bits = int(((v * _H01) >> 24).sum())
|
||||
return 1.0 - diff_bits / (n * 32.0)
|
||||
|
||||
|
||||
def _compare_fingerprints(fp1: np.ndarray, fp2: np.ndarray, max_offset: int = MAX_OFFSET) -> float:
|
||||
if fp1.size < 50 or fp2.size < 50:
|
||||
return 0.0
|
||||
# Fast pre-filter: try offset 0 first. If unrelated, skip the offset
|
||||
# search entirely. Same recordings cluster within a few offsets of 0,
|
||||
# so a sub-0.50 base score is a definitive "not a duplicate."
|
||||
base = _sim_at_offset(fp1, fp2, 0)
|
||||
if base < 0.50:
|
||||
return base
|
||||
best = base
|
||||
if best >= 0.99:
|
||||
return best
|
||||
for offset in range(1, max_offset + 1):
|
||||
for o in (offset, -offset):
|
||||
sim = _sim_at_offset(fp1, fp2, o)
|
||||
if sim > best:
|
||||
best = sim
|
||||
if best >= 0.99:
|
||||
return best
|
||||
return best
|
||||
|
||||
|
||||
_DESKTOP_RE = re.compile(r"-DESKTOP-[A-Z0-9]+", re.IGNORECASE)
|
||||
_DUP_SUFFIX_RE = re.compile(r"\(\d+\)\.[a-z]+$|\.\d+\.[a-z]+$", re.IGNORECASE)
|
||||
# DJ/club-friendly versions to prefer when same audio matches at different
|
||||
# labellings. Bonus is small enough not to override format/size when those
|
||||
# signal a real quality difference.
|
||||
_EXTENDED_RE = re.compile(
|
||||
r"\b(extended|club|dj edit|dj mix|long version|original mix)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def rank_file(host_path: str) -> int:
|
||||
"""Lower is better. FLAC > non-FLAC; clean filename > sync-conflict
|
||||
artifact; extended/club mixes beat radio edits at same format; then
|
||||
larger wins. Penalty weights:
|
||||
DESKTOP-suffix (OneDrive sync conflict) > (N)/.N. (Windows/beets dup)
|
||||
so an exclusively-(1)-marked file beats one that's both (1) and DESKTOP."""
|
||||
try:
|
||||
size = os.path.getsize(host_path)
|
||||
except OSError:
|
||||
size = 0
|
||||
ext_score = 1 if host_path.lower().endswith(".flac") else 0
|
||||
desktop = 1 if _DESKTOP_RE.search(host_path) else 0
|
||||
dup = 1 if _DUP_SUFFIX_RE.search(host_path) else 0
|
||||
extended = 1 if _EXTENDED_RE.search(host_path) else 0
|
||||
return (
|
||||
10_000_000_000
|
||||
- ext_score * 1_000_000_000_000
|
||||
+ desktop * 500_000_000_000
|
||||
+ dup * 100_000_000_000
|
||||
- extended * 50_000_000_000
|
||||
- size
|
||||
)
|
||||
|
||||
|
||||
class UnionFind:
|
||||
def __init__(self, ids):
|
||||
self.parent = {i: i for i in ids}
|
||||
|
||||
def find(self, x):
|
||||
while self.parent[x] != x:
|
||||
self.parent[x] = self.parent[self.parent[x]]
|
||||
x = self.parent[x]
|
||||
return x
|
||||
|
||||
def union(self, a, b):
|
||||
ra, rb = self.find(a), self.find(b)
|
||||
if ra != rb:
|
||||
self.parent[ra] = rb
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--apply", action="store_true", help="actually delete losers")
|
||||
ap.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD)
|
||||
ap.add_argument("--duration-tol", type=int, default=DURATION_TOLERANCE)
|
||||
ap.add_argument("--no-cache", action="store_true",
|
||||
help="ignore the scan cache and re-fingerprint everything")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not os.path.exists(FINGERPRINT_DB):
|
||||
print(f"ERROR: fingerprint index missing at {FINGERPRINT_DB}", file=sys.stderr)
|
||||
print("Run build-fingerprint-index.py first.", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
conn = sqlite3.connect(f"file:{FINGERPRINT_DB}?mode=ro", uri=True)
|
||||
rows = list(conn.execute("SELECT beets_id, path, duration, fingerprint FROM fingerprints"))
|
||||
conn.close()
|
||||
|
||||
entries = []
|
||||
for beets_id, path, duration, fp_csv in rows:
|
||||
fp = np.fromstring(fp_csv, dtype=np.uint32, sep=",")
|
||||
if fp.size == 0:
|
||||
continue
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
entries.append((beets_id, path, duration, fp))
|
||||
print(f"[fuzzy-dupes] loaded {len(entries)} fingerprints (skipped missing/empty)")
|
||||
|
||||
# Bucket by duration to make pairwise scan O(N * window) instead of O(N^2).
|
||||
by_dur = defaultdict(list)
|
||||
for idx, (_, _, dur, _) in enumerate(entries):
|
||||
by_dur[dur].append(idx)
|
||||
|
||||
# Try cache first. Cache key = (fingerprint mtime, threshold, duration_tol).
|
||||
fp_mtime = os.path.getmtime(FINGERPRINT_DB)
|
||||
cache_key = {"fp_mtime": fp_mtime, "threshold": args.threshold,
|
||||
"duration_tol": args.duration_tol, "n_entries": len(entries)}
|
||||
pair_scores: dict[tuple[int, int], float] = {}
|
||||
if not args.no_cache and os.path.exists(SCAN_CACHE):
|
||||
try:
|
||||
with open(SCAN_CACHE) as f:
|
||||
cache = json.load(f)
|
||||
if cache.get("key") == cache_key:
|
||||
pair_scores = {tuple(map(int, k.split(","))): v
|
||||
for k, v in cache["pairs"].items()}
|
||||
print(f"[fuzzy-dupes] loaded {len(pair_scores)} pair scores from cache "
|
||||
f"({SCAN_CACHE})")
|
||||
except (OSError, json.JSONDecodeError, KeyError):
|
||||
pair_scores = {}
|
||||
|
||||
if not pair_scores:
|
||||
start = time.time()
|
||||
compared = 0
|
||||
matched = 0
|
||||
for i, (_, path_i, dur_i, fp_i) in enumerate(entries):
|
||||
for d in range(dur_i - args.duration_tol, dur_i + args.duration_tol + 1):
|
||||
for j in by_dur.get(d, []):
|
||||
if j <= i:
|
||||
continue
|
||||
compared += 1
|
||||
fp_j = entries[j][3]
|
||||
score = _compare_fingerprints(fp_i, fp_j)
|
||||
if score >= args.threshold:
|
||||
pair_scores[(i, j)] = score
|
||||
matched += 1
|
||||
if (i + 1) % 250 == 0:
|
||||
elapsed = time.time() - start
|
||||
rate = (i + 1) / elapsed if elapsed > 0 else 0
|
||||
print(f"[fuzzy-dupes] scanned {i + 1}/{len(entries)} "
|
||||
f"({rate:.1f}/s, {compared} compared, {matched} matches)", flush=True)
|
||||
try:
|
||||
with open(SCAN_CACHE, "w") as f:
|
||||
json.dump({
|
||||
"key": cache_key,
|
||||
"pairs": {f"{i},{j}": s for (i, j), s in pair_scores.items()},
|
||||
}, f)
|
||||
print(f"[fuzzy-dupes] cached {len(pair_scores)} pair scores → {SCAN_CACHE}")
|
||||
except OSError as e:
|
||||
print(f"[fuzzy-dupes] WARN: could not write cache: {e}", file=sys.stderr)
|
||||
|
||||
uf = UnionFind(range(len(entries)))
|
||||
for (i, j) in pair_scores:
|
||||
uf.union(i, j)
|
||||
|
||||
# Group by union-find root.
|
||||
groups: dict[int, list[int]] = defaultdict(list)
|
||||
for idx in range(len(entries)):
|
||||
groups[uf.find(idx)].append(idx)
|
||||
|
||||
dup_groups = [g for g in groups.values() if len(g) >= 2]
|
||||
print(f"\n[fuzzy-dupes] found {len(dup_groups)} duplicate groups "
|
||||
f"covering {sum(len(g) for g in dup_groups)} files\n")
|
||||
|
||||
if not dup_groups:
|
||||
return 0
|
||||
|
||||
delete_targets: list[tuple[int, str]] = []
|
||||
skipped_transitive = 0
|
||||
for group in sorted(dup_groups, key=lambda g: -len(g)):
|
||||
ranked = sorted(group, key=lambda idx: rank_file(entries[idx][1]))
|
||||
keeper_idx = ranked[0]
|
||||
keeper_path = entries[keeper_idx][1]
|
||||
keeper_fp = entries[keeper_idx][3]
|
||||
keeper_size = os.path.getsize(keeper_path)
|
||||
keeper_ext = keeper_path.rsplit(".", 1)[-1].upper()
|
||||
print(f"--- group ({len(group)} files) ---")
|
||||
print(f" KEEP [{keeper_ext} {keeper_size / 1024 / 1024:.1f}M] {keeper_path}")
|
||||
for loser_idx in ranked[1:]:
|
||||
beets_id, loser_path, _, loser_fp = entries[loser_idx]
|
||||
loser_size = os.path.getsize(loser_path) if os.path.exists(loser_path) else 0
|
||||
loser_ext = loser_path.rsplit(".", 1)[-1].upper()
|
||||
# Verify direct similarity to the keeper. Transitive union-find can
|
||||
# pull in unrelated tracks via a chain of partial matches (e.g. a
|
||||
# game-OST jingle group where A~B and B~C but A and C are unrelated).
|
||||
pair_key = (min(keeper_idx, loser_idx), max(keeper_idx, loser_idx))
|
||||
score = pair_scores.get(pair_key)
|
||||
if score is None:
|
||||
score = _compare_fingerprints(keeper_fp, loser_fp)
|
||||
if score < args.threshold:
|
||||
print(f" SKIP [{loser_ext} {loser_size / 1024 / 1024:.1f}M sim={score:.3f} to keeper] {loser_path}")
|
||||
skipped_transitive += 1
|
||||
continue
|
||||
print(f" DELETE [{loser_ext} {loser_size / 1024 / 1024:.1f}M sim={score:.3f}] {loser_path}")
|
||||
delete_targets.append((beets_id, loser_path))
|
||||
print()
|
||||
if skipped_transitive:
|
||||
print(f"[fuzzy-dupes] skipped {skipped_transitive} transitive false-positives "
|
||||
f"(chain-grouped but sim<{args.threshold} to keeper)\n")
|
||||
|
||||
if not args.apply:
|
||||
print(f"\n[fuzzy-dupes] DRY RUN — pass --apply to delete {len(delete_targets)} files")
|
||||
return 0
|
||||
|
||||
print(f"\n[fuzzy-dupes] APPLY: deleting {len(delete_targets)} files via beet remove -d -f")
|
||||
failed = 0
|
||||
for beets_id, path in delete_targets:
|
||||
# Escape regex metacharacters for path:: regex query
|
||||
escaped = re.escape(path)
|
||||
result = subprocess.run(
|
||||
["beet", "remove", "-d", "-f", f"path::{escaped}"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f" FAILED: {path} — {result.stderr.strip()}", file=sys.stderr)
|
||||
failed += 1
|
||||
print(f"[fuzzy-dupes] done. {len(delete_targets) - failed} deleted, {failed} failed.")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/bin/bash
|
||||
# Find audio files in the library that have no ALBUM tag, set ALBUM to
|
||||
# "{TITLE} - Single" — matches Spotify / Apple Music's convention for
|
||||
# single-track releases. Otherwise these tracks land as [Unknown Album]
|
||||
# in Navidrome.
|
||||
#
|
||||
# Usage:
|
||||
# fix-empty-album.sh # dry run (default)
|
||||
# fix-empty-album.sh --apply # actually rewrite tags
|
||||
#
|
||||
# Bandcamp single-track downloads are the most common source of this
|
||||
# issue (Bandcamp serves single tracks with no ALBUM tag). New downloads
|
||||
# via sync-bandcamp.py are auto-tagged; this script handles backfill.
|
||||
|
||||
set -u
|
||||
APPLY=0
|
||||
[[ "${1:-}" == "--apply" ]] && APPLY=1
|
||||
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/fix-empty-album-$(date +%Y%m%d-%H%M%S).log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
NAVIDROME_ENV="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/navidrome/admin.env"
|
||||
[ -f "$NAVIDROME_ENV" ] && source "$NAVIDROME_ENV"
|
||||
ND_BASE="${ND_BASE:-http://navidrome:4533}"
|
||||
ND_USER="${ND_USER:-andrew}"
|
||||
ND_PASS="${ND_PASS:-}"
|
||||
log() { echo "[$(date -Iseconds)] $*" | tee -a "$LOG"; }
|
||||
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
log "=== FIX EMPTY ALBUM (APPLY) ==="
|
||||
else
|
||||
log "=== FIX EMPTY ALBUM (DRY RUN — pass --apply to write) ==="
|
||||
fi
|
||||
|
||||
FIXED=0
|
||||
SKIPPED=0
|
||||
|
||||
while IFS= read -r -d '' f; do
|
||||
ext="${f,,}"; ext="${ext##*.}"
|
||||
album="" ; title=""
|
||||
case "$ext" in
|
||||
flac)
|
||||
album=$(metaflac --show-tag=ALBUM "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
title=$(metaflac --show-tag=TITLE "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
;;
|
||||
mp3)
|
||||
album=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TALB[^:]*: //p' | head -1)
|
||||
title=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TIT2[^:]*: //p' | head -1)
|
||||
;;
|
||||
*) continue ;;
|
||||
esac
|
||||
|
||||
if [[ -n "$album" ]]; then
|
||||
continue # already has an album, leave it alone
|
||||
fi
|
||||
if [[ -z "$title" ]]; then
|
||||
log "[skip-no-title] $f"
|
||||
SKIPPED=$((SKIPPED + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Some titles have a leading "NN " track-number prefix baked in from a
|
||||
# bad filename parse (e.g. title="01 Domestic Violence"). If we use the
|
||||
# title as-is, the album becomes "01 Domestic Violence - Single" which
|
||||
# is broken. Strip leading 1-3 digits + space, and also fix the title
|
||||
# tag in place so it isn't shown with the prefix in clients.
|
||||
clean_title=$(echo "$title" | sed -E 's/^[[:space:]]*[0-9]{1,3}[[:space:]]+//')
|
||||
if [[ "$clean_title" != "$title" && -n "$clean_title" ]]; then
|
||||
log " trimmed leading track-number from title: \"$title\" -> \"$clean_title\""
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
case "$ext" in
|
||||
flac) metaflac --remove-tag=TITLE --set-tag="TITLE=${clean_title}" "$f" 2>>"$LOG" ;;
|
||||
mp3) id3v2 --TIT2 "${clean_title}" "$f" 2>>"$LOG" ;;
|
||||
esac
|
||||
fi
|
||||
title="$clean_title"
|
||||
fi
|
||||
|
||||
new_album="${title} - Single"
|
||||
log "[$( [[ $APPLY -eq 1 ]] && echo SET || echo WOULD-SET )] album=\"${new_album}\" - $f"
|
||||
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
case "$ext" in
|
||||
flac) metaflac --set-tag="ALBUM=${new_album}" "$f" 2>>"$LOG" ;;
|
||||
mp3) id3v2 --TALB "${new_album}" "$f" 2>>"$LOG" ;;
|
||||
esac
|
||||
fi
|
||||
FIXED=$((FIXED + 1))
|
||||
done < <(find ${MUSIC_DATA_DIR:-/data/music}/Library -type f \( -iname "*.flac" -o -iname "*.mp3" \) -print0 2>/dev/null)
|
||||
|
||||
log ""
|
||||
log "=== Summary: $FIXED fixed, $SKIPPED skipped (no title) ==="
|
||||
|
||||
if [[ $APPLY -eq 1 && $FIXED -gt 0 ]]; then
|
||||
log "Syncing beets DB from new tags..."
|
||||
beet update 2>&1 | tail -3 >> "$LOG"
|
||||
log "Moving files to reflect new album in path templates..."
|
||||
beet move 2>&1 | tail -3 >> "$LOG"
|
||||
log "Triggering Navidrome rescan..."
|
||||
curl -s -G "$ND_BASE/rest/startScan.view" \
|
||||
--data-urlencode "u=$ND_USER" \
|
||||
--data-urlencode "p=$ND_PASS" \
|
||||
--data-urlencode 'v=1.16.0' \
|
||||
--data-urlencode 'c=fix-empty-album' \
|
||||
--data-urlencode 'f=json' \
|
||||
--data-urlencode 'fullScan=true' >> "$LOG" 2>&1
|
||||
fi
|
||||
|
||||
log "=== Done. Log: $LOG ==="
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/bin/bash
|
||||
# fix-genre.sh — quickly override the GENRE tag for every track by an artist.
|
||||
#
|
||||
# Use when spotify-genre.py picks something wrong (e.g. it tagged Yasuha as
|
||||
# "Asian Music" when "Anime; Soundtrack" is what you actually want), or when
|
||||
# you want to set a genre on an artist Spotify doesn't have data for.
|
||||
#
|
||||
# The new value is written to file tags AND beets's DB. The next nightly
|
||||
# spotify-genre run will leave these alone (they already match the whitelist
|
||||
# format), unless you pass --force.
|
||||
#
|
||||
# Usage:
|
||||
# fix-genre.sh "<artist>" "<Genre1; Genre2; ...>" # apply
|
||||
# fix-genre.sh --dry "<artist>" "<Genre1; ...>" # preview only
|
||||
#
|
||||
# Examples:
|
||||
# sudo fix-genre.sh "Yasuha" "Anime; Soundtrack"
|
||||
# sudo fix-genre.sh "Aphex Twin" "IDM; Electronic; Ambient"
|
||||
# sudo fix-genre.sh --dry "Mac DeMarco" "Indie; Indie Rock"
|
||||
|
||||
set -u
|
||||
|
||||
DRY=0
|
||||
if [[ "${1:-}" == "--dry" ]]; then
|
||||
DRY=1
|
||||
shift
|
||||
fi
|
||||
|
||||
ARTIST="${1:?Usage: $0 [--dry] <artist> <Genre1; Genre2; ...>}"
|
||||
GENRE="${2:?Usage: $0 [--dry] <artist> <Genre1; Genre2; ...>}"
|
||||
|
||||
LIBRARY=${MUSIC_DATA_DIR:-/data/music}/Library
|
||||
|
||||
# Pull paths via beets (in-process, same container).
|
||||
mapfile -t paths < <(beet ls -f '$path' "artist:$ARTIST" 2>/dev/null)
|
||||
|
||||
if [[ "${#paths[@]}" -eq 0 ]]; then
|
||||
echo "[fix-genre] no tracks found in beets for artist:$ARTIST"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[fix-genre] artist=\"$ARTIST\""
|
||||
echo "[fix-genre] new GENRE=\"$GENRE\""
|
||||
echo "[fix-genre] ${#paths[@]} track(s) match"
|
||||
[[ $DRY -eq 1 ]] && echo "[fix-genre] DRY RUN — no tags will be written" && exit 0
|
||||
|
||||
ok=0
|
||||
fail=0
|
||||
for cp in "${paths[@]}"; do
|
||||
hp="${LIBRARY}${cp#/music}"
|
||||
if [[ ! -f "$hp" ]]; then
|
||||
echo " MISS $hp"
|
||||
fail=$((fail + 1))
|
||||
continue
|
||||
fi
|
||||
case "${hp,,}" in
|
||||
*.flac)
|
||||
metaflac --remove-tag=GENRE "$hp" 2>/dev/null
|
||||
metaflac --set-tag="GENRE=$GENRE" "$hp" 2>/dev/null && ok=$((ok + 1)) || fail=$((fail + 1))
|
||||
metaflac --remove-tag=GENRE_LOCK "$hp" 2>/dev/null
|
||||
metaflac --set-tag="GENRE_LOCK=1" "$hp" 2>/dev/null
|
||||
;;
|
||||
*.mp3)
|
||||
# mid3v2 / id3v2 don't have a clean way to set TCON via plain id3v2 here,
|
||||
# so use mutagen via a one-liner.
|
||||
python3 - "$hp" "$GENRE" <<'PY' && ok=$((ok + 1)) || fail=$((fail + 1))
|
||||
import sys
|
||||
from mutagen.id3 import ID3, TCON, TXXX, ID3NoHeaderError
|
||||
path, g = sys.argv[1], sys.argv[2]
|
||||
try:
|
||||
t = ID3(path)
|
||||
except ID3NoHeaderError:
|
||||
t = ID3()
|
||||
t.delall("TCON")
|
||||
t.add(TCON(encoding=3, text=g))
|
||||
t.delall("TXXX:GENRE_LOCK")
|
||||
t.add(TXXX(encoding=3, desc="GENRE_LOCK", text="1"))
|
||||
t.save(path)
|
||||
PY
|
||||
;;
|
||||
*.m4a|*.mp4)
|
||||
python3 - "$hp" "$GENRE" <<'PY' && ok=$((ok + 1)) || fail=$((fail + 1))
|
||||
import sys
|
||||
from mutagen.mp4 import MP4
|
||||
path, g = sys.argv[1], sys.argv[2]
|
||||
m = MP4(path)
|
||||
m["\xa9gen"] = g
|
||||
m.save()
|
||||
PY
|
||||
;;
|
||||
*.opus|*.ogg)
|
||||
python3 - "$hp" "$GENRE" <<'PY' && ok=$((ok + 1)) || fail=$((fail + 1))
|
||||
import sys
|
||||
from mutagen import File as MFile
|
||||
path, g = sys.argv[1], sys.argv[2]
|
||||
m = MFile(path)
|
||||
m.tags["genre"] = [g]
|
||||
m.save()
|
||||
PY
|
||||
;;
|
||||
*)
|
||||
echo " SKIP unsupported format: $hp"
|
||||
fail=$((fail + 1))
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "[fix-genre] tagged ok=$ok, failed=$fail"
|
||||
|
||||
# Sync beets DB so its $genres column matches the file tags.
|
||||
# `-q` is NOT a flag for `beet update` (only `beet ls`); leaving it off
|
||||
# lets beets actually apply the diff non-interactively.
|
||||
echo "[fix-genre] syncing beets DB"
|
||||
beet update "artist:$ARTIST" 2>&1 | grep -E '^\s+[+-]' || true
|
||||
|
||||
# Verify
|
||||
echo "[fix-genre] new state:"
|
||||
beet ls -f ' $title — $genres' "artist:$ARTIST" | head -5
|
||||
remaining=$(beet ls "artist:$ARTIST" | wc -l)
|
||||
shown=5
|
||||
[[ "$remaining" -gt "$shown" ]] && echo " ...and $((remaining - shown)) more"
|
||||
|
||||
echo "[fix-genre] done. Trigger a Navidrome scan to surface the change in the UI."
|
||||
Executable
+514
@@ -0,0 +1,514 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
fix-track-metadata.py — interactively rewrite a single track's metadata
|
||||
from a Spotify or iTunes URL.
|
||||
|
||||
Prompts for an audio file path and a Spotify/iTunes URL, then:
|
||||
1. Fetches canonical metadata (artist, album, title, track#, date, cover).
|
||||
2. Writes tags to the file (artist semicolon-joined, albumartist primary
|
||||
only — matches the rest of the booksandtunes pipeline).
|
||||
3. Strips SoundCloud / upload-junk frames (WWWAUDIOFILE, comment,
|
||||
compatible_brands, major_brand, minor_version).
|
||||
4. Replaces embedded cover with the URL's artwork and drops a cover.jpg
|
||||
sidecar in the album folder.
|
||||
5. Runs `beet update` so beets's DB matches.
|
||||
6. Runs `beet move` to relocate the file under
|
||||
/music/<albumartist>/<album>/<title>.<ext>.
|
||||
7. Strips GENRE and runs spotify-genre.py to refill from the artist's
|
||||
Spotify per-artist genres (use --keep-genre to skip this).
|
||||
8. Triggers a Navidrome rescan via the Subsonic API.
|
||||
|
||||
Usage:
|
||||
./fix-track-metadata.py # interactive
|
||||
./fix-track-metadata.py <file> <url> # non-interactive
|
||||
./fix-track-metadata.py --keep-genre <f> <u> # leave existing GENRE alone
|
||||
|
||||
The <file> arg accepts any of:
|
||||
- absolute host path: $MUSIC_DATA_DIR/Library/Foo/Bar.flac
|
||||
- beets container path: /music/Foo/Bar.flac
|
||||
- library-relative path: Foo/Bar.flac
|
||||
(resolved against $MUSIC_DATA_DIR/Library/)
|
||||
|
||||
Supported URLs:
|
||||
https://open.spotify.com/track/<id>...
|
||||
https://open.spotify.com/album/<id>... (single-track albums only)
|
||||
https://music.apple.com/.../id<col>?i=<track> (iTunes track URL)
|
||||
"""
|
||||
import sys, os, re, json, base64, argparse, subprocess, unicodedata, urllib.parse, urllib.request
|
||||
|
||||
from mutagen import File as MFile
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError, TPE1, TPE2, TALB, TIT2, TRCK, TDRC, TCON, APIC
|
||||
from mutagen.flac import FLAC, Picture
|
||||
from mutagen.mp4 import MP4, MP4Cover
|
||||
from mutagen.oggopus import OggOpus
|
||||
from mutagen.oggvorbis import OggVorbis
|
||||
|
||||
_ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
|
||||
_MUSIC_DATA_DIR = os.environ.get("MUSIC_DATA_DIR", "/data/music")
|
||||
|
||||
LIBRARY = f"{_MUSIC_DATA_DIR}/Library"
|
||||
# Still "/music" through the migration's Stage 0-3 transitional beets mount;
|
||||
# revisit at Stage 4 once beets' directory: becomes MUSIC_DATA_DIR/Library.
|
||||
CONTAINER_PREFIX = "/music"
|
||||
SPOTIFY_ENV = f"{_ALEMBIC_CONFIG_DIR}/pipeline/_spotify.env"
|
||||
SPOTIFY_GENRE = f"{os.environ.get('PIPELINE_DIR', '/app/pipeline')}/lib/spotify-genre.py"
|
||||
|
||||
|
||||
def _load_navidrome_env():
|
||||
path = f"{_ALEMBIC_CONFIG_DIR}/pipeline/navidrome/admin.env"
|
||||
env = {}
|
||||
if os.path.exists(path):
|
||||
for line in open(path):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env[k] = v.strip().strip("'").strip('"')
|
||||
return env
|
||||
|
||||
_nd_env = _load_navidrome_env()
|
||||
NAVIDROME_URL = f"{_nd_env.get('ND_BASE', 'http://navidrome:4533')}/rest/startScan.view"
|
||||
NAVIDROME_USER = _nd_env.get("ND_USER", "andrew")
|
||||
NAVIDROME_PASS = _nd_env.get("ND_PASS", "")
|
||||
|
||||
VORBIS_LIKE = (FLAC, OggOpus, OggVorbis)
|
||||
JUNK_TXXX = {"WWWAUDIOFILE", "comment", "compatible_brands",
|
||||
"major_brand", "minor_version"}
|
||||
|
||||
|
||||
# ---------- Spotify ----------
|
||||
|
||||
def _spotify_token():
|
||||
env = {}
|
||||
with open(SPOTIFY_ENV) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env[k] = v.strip().strip("'").strip('"')
|
||||
cid, csec = env["SPOTIFY_CLIENT_ID"], env["SPOTIFY_CLIENT_SECRET"]
|
||||
creds = base64.b64encode(f"{cid}:{csec}".encode()).decode()
|
||||
req = urllib.request.Request(
|
||||
"https://accounts.spotify.com/api/token",
|
||||
data=urllib.parse.urlencode({"grant_type": "client_credentials"}).encode(),
|
||||
headers={"Authorization": f"Basic {creds}",
|
||||
"Content-Type": "application/x-www-form-urlencoded"})
|
||||
return json.loads(urllib.request.urlopen(req, timeout=15).read())["access_token"]
|
||||
|
||||
|
||||
def _spotify_get(path, token):
|
||||
req = urllib.request.Request(f"https://api.spotify.com/v1{path}",
|
||||
headers={"Authorization": f"Bearer {token}"})
|
||||
return json.loads(urllib.request.urlopen(req, timeout=15).read())
|
||||
|
||||
|
||||
def fetch_spotify_track(track_id):
|
||||
tok = _spotify_token()
|
||||
t = _spotify_get(f"/tracks/{track_id}", tok)
|
||||
a = t["album"]
|
||||
return {
|
||||
"title": t["name"],
|
||||
"artists": [ar["name"] for ar in t["artists"]],
|
||||
"album": a["name"],
|
||||
"albumartists": [ar["name"] for ar in a["artists"]],
|
||||
"date": a["release_date"],
|
||||
"track": t["track_number"],
|
||||
"tracktotal": a["total_tracks"],
|
||||
"cover_url": a["images"][0]["url"] if a["images"] else None,
|
||||
}
|
||||
|
||||
|
||||
def fetch_spotify_album(album_id):
|
||||
tok = _spotify_token()
|
||||
a = _spotify_get(f"/albums/{album_id}", tok)
|
||||
if a["total_tracks"] != 1:
|
||||
raise SystemExit(f"album has {a['total_tracks']} tracks — pass a track URL instead")
|
||||
t = a["tracks"]["items"][0]
|
||||
return {
|
||||
"title": t["name"],
|
||||
"artists": [ar["name"] for ar in t["artists"]],
|
||||
"album": a["name"],
|
||||
"albumartists": [ar["name"] for ar in a["artists"]],
|
||||
"date": a["release_date"],
|
||||
"track": 1,
|
||||
"tracktotal": 1,
|
||||
"cover_url": a["images"][0]["url"] if a["images"] else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------- iTunes ----------
|
||||
|
||||
def fetch_itunes_track(track_id):
|
||||
url = f"https://itunes.apple.com/lookup?id={track_id}&entity=song"
|
||||
data = json.loads(urllib.request.urlopen(url, timeout=15).read())
|
||||
items = [r for r in data.get("results", []) if r.get("wrapperType") == "track"]
|
||||
if not items:
|
||||
raise SystemExit(f"iTunes lookup returned no track for id={track_id}")
|
||||
r = items[0]
|
||||
# iTunes only gives one artist string — the project convention is
|
||||
# semicolon-joined for ARTIST. The user can split manually after if needed.
|
||||
artists = [r["artistName"]]
|
||||
cover = r.get("artworkUrl100", "")
|
||||
if cover:
|
||||
# 100x100 → 600x600 by URL substitution (well-known iTunes hack)
|
||||
cover = re.sub(r"/\d+x\d+bb\.jpg$", "/600x600bb.jpg", cover)
|
||||
return {
|
||||
"title": r["trackName"],
|
||||
"artists": artists,
|
||||
"album": r["collectionName"],
|
||||
"albumartists": artists,
|
||||
"date": (r.get("releaseDate") or "")[:10],
|
||||
"track": r.get("trackNumber", 1),
|
||||
"tracktotal": r.get("trackCount", 1),
|
||||
"cover_url": cover or None,
|
||||
}
|
||||
|
||||
|
||||
# ---------- URL dispatch ----------
|
||||
|
||||
def fetch_metadata(url):
|
||||
m = re.search(r"open\.spotify\.com/track/([A-Za-z0-9]+)", url)
|
||||
if m:
|
||||
return fetch_spotify_track(m.group(1))
|
||||
m = re.search(r"open\.spotify\.com/album/([A-Za-z0-9]+)", url)
|
||||
if m:
|
||||
return fetch_spotify_album(m.group(1))
|
||||
m = re.search(r"music\.apple\.com/.+?[?&]i=(\d+)", url)
|
||||
if m:
|
||||
return fetch_itunes_track(m.group(1))
|
||||
m = re.search(r"music\.apple\.com/.+?/song/[^/]+/(\d+)", url)
|
||||
if m:
|
||||
return fetch_itunes_track(m.group(1))
|
||||
raise SystemExit(f"unrecognized URL: {url}")
|
||||
|
||||
|
||||
# ---------- Tagging ----------
|
||||
|
||||
def write_tags(filepath, meta):
|
||||
"""Write artist/albumartist/album/title/track/date and embedded cover.
|
||||
Drops watermark/junk TXXX frames as a side effect.
|
||||
"""
|
||||
artist_joined = "; ".join(meta["artists"])
|
||||
albumartist = meta["albumartists"][0] if meta["albumartists"] else meta["artists"][0]
|
||||
cover_bytes = _download_cover(meta["cover_url"]) if meta["cover_url"] else None
|
||||
date = meta["date"] # YYYY-MM-DD or YYYY
|
||||
|
||||
m = MFile(filepath)
|
||||
if m is None:
|
||||
raise SystemExit(f"unsupported audio format: {filepath}")
|
||||
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
_write_vorbis(m, artist_joined, albumartist, meta, date, cover_bytes)
|
||||
elif isinstance(m, MP4):
|
||||
_write_mp4(m, artist_joined, albumartist, meta, date, cover_bytes)
|
||||
else:
|
||||
_write_id3(filepath, artist_joined, albumartist, meta, date, cover_bytes)
|
||||
|
||||
|
||||
def _write_vorbis(m, artist_joined, albumartist, meta, date, cover_bytes):
|
||||
m.tags["ARTIST"] = artist_joined
|
||||
m.tags["ALBUMARTIST"] = albumartist
|
||||
m.tags["ALBUM"] = meta["album"]
|
||||
m.tags["TITLE"] = meta["title"]
|
||||
m.tags["TRACKNUMBER"] = str(meta["track"])
|
||||
m.tags["TRACKTOTAL"] = str(meta["tracktotal"])
|
||||
m.tags["DATE"] = date
|
||||
if cover_bytes and isinstance(m, FLAC):
|
||||
m.clear_pictures()
|
||||
pic = Picture()
|
||||
pic.type = 3 # Cover (front)
|
||||
pic.mime = "image/jpeg"
|
||||
pic.data = cover_bytes
|
||||
m.add_picture(pic)
|
||||
m.save()
|
||||
|
||||
|
||||
def _write_mp4(m, artist_joined, albumartist, meta, date, cover_bytes):
|
||||
m.tags["\xa9ART"] = artist_joined
|
||||
m.tags["aART"] = albumartist
|
||||
m.tags["\xa9alb"] = meta["album"]
|
||||
m.tags["\xa9nam"] = meta["title"]
|
||||
m.tags["\xa9day"] = date
|
||||
m.tags["trkn"] = [(int(meta["track"]), int(meta["tracktotal"]))]
|
||||
if cover_bytes:
|
||||
m.tags["covr"] = [MP4Cover(cover_bytes, imageformat=MP4Cover.FORMAT_JPEG)]
|
||||
m.save()
|
||||
|
||||
|
||||
def _write_id3(filepath, artist_joined, albumartist, meta, date, cover_bytes):
|
||||
try:
|
||||
t = ID3(filepath)
|
||||
except ID3NoHeaderError:
|
||||
t = ID3()
|
||||
t.delall("TPE1"); t.add(TPE1(encoding=3, text=artist_joined))
|
||||
t.delall("TPE2"); t.add(TPE2(encoding=3, text=albumartist))
|
||||
t.delall("TALB"); t.add(TALB(encoding=3, text=meta["album"]))
|
||||
t.delall("TIT2"); t.add(TIT2(encoding=3, text=meta["title"]))
|
||||
t.delall("TRCK"); t.add(TRCK(encoding=3, text=f"{meta['track']}/{meta['tracktotal']}"))
|
||||
t.delall("TDRC"); t.add(TDRC(encoding=3, text=date))
|
||||
# Strip junk TXXX frames
|
||||
for frame in list(t.getall("TXXX")):
|
||||
if frame.desc in JUNK_TXXX:
|
||||
t.delall(f"TXXX:{frame.desc}")
|
||||
if cover_bytes:
|
||||
t.delall("APIC")
|
||||
t.add(APIC(encoding=3, mime="image/jpeg", type=3, desc="Cover", data=cover_bytes))
|
||||
t.save(filepath)
|
||||
|
||||
|
||||
def _download_cover(url):
|
||||
if not url:
|
||||
return None
|
||||
return urllib.request.urlopen(url, timeout=20).read()
|
||||
|
||||
|
||||
# ---------- Beets / Navidrome ----------
|
||||
|
||||
def host_to_container(host_path):
|
||||
if host_path.startswith(LIBRARY):
|
||||
return CONTAINER_PREFIX + host_path[len(LIBRARY):]
|
||||
return host_path
|
||||
|
||||
|
||||
def container_to_host(cp):
|
||||
if cp.startswith(CONTAINER_PREFIX + "/"):
|
||||
return LIBRARY + cp[len(CONTAINER_PREFIX):]
|
||||
return cp
|
||||
|
||||
|
||||
def resolve_input_path(arg):
|
||||
"""Accept a host path, container path, or library-relative path; return
|
||||
the absolute host path that actually exists on disk.
|
||||
|
||||
Falls back through NFC/NFD Unicode normalizations because terminal pastes
|
||||
of Japanese/Korean/etc. paths often come in decomposed form even when the
|
||||
filesystem stores them composed (or vice versa)."""
|
||||
arg = arg.strip().strip('"').strip("'")
|
||||
if arg.startswith(CONTAINER_PREFIX + "/"):
|
||||
candidate = container_to_host(arg)
|
||||
elif os.path.isabs(arg):
|
||||
candidate = arg
|
||||
else:
|
||||
candidate = os.path.join(LIBRARY, arg)
|
||||
|
||||
if os.path.exists(candidate):
|
||||
return candidate
|
||||
for form in ("NFC", "NFD", "NFKC", "NFKD"):
|
||||
norm = unicodedata.normalize(form, candidate)
|
||||
if os.path.exists(norm):
|
||||
return norm
|
||||
# Last resort: walk component by component, picking the entry whose NFC
|
||||
# form matches our NFC form. Handles the common case where one directory
|
||||
# in the path differs in normalization but the rest is fine.
|
||||
if not candidate.startswith("/"):
|
||||
return candidate
|
||||
parts = candidate.lstrip("/").split("/")
|
||||
cur = "/"
|
||||
for want in parts:
|
||||
if not os.path.isdir(cur):
|
||||
return candidate
|
||||
want_nfc = unicodedata.normalize("NFC", want)
|
||||
match = next(
|
||||
(n for n in os.listdir(cur)
|
||||
if unicodedata.normalize("NFC", n) == want_nfc),
|
||||
None)
|
||||
if match is None:
|
||||
return candidate
|
||||
cur = os.path.join(cur, match)
|
||||
return cur
|
||||
|
||||
|
||||
def beets_id_for(host_path):
|
||||
cp = host_to_container(host_path)
|
||||
r = subprocess.run(
|
||||
["beet", "ls", "-f", "$id|||$path", f"path:{cp}"],
|
||||
capture_output=True, text=True, check=True)
|
||||
for line in r.stdout.splitlines():
|
||||
if "|||" in line:
|
||||
id_str, p = line.split("|||", 1)
|
||||
if p == cp:
|
||||
return int(id_str)
|
||||
return None
|
||||
|
||||
|
||||
def beet_update(beets_id):
|
||||
# `beet update` prompts y/n on each diff; `yes y` auto-confirms.
|
||||
subprocess.run(
|
||||
f"yes y | beet update id:{beets_id}",
|
||||
shell=True, check=False)
|
||||
|
||||
|
||||
def beet_move(beets_id):
|
||||
"""Beets path-format relocates the file under LIBRARY/<albumartist>/<album>/."""
|
||||
subprocess.run(["beet", "move", f"id:{beets_id}"], check=False)
|
||||
|
||||
|
||||
def beet_path(beets_id):
|
||||
r = subprocess.run(
|
||||
["beet", "ls", "-f", "$path", f"id:{beets_id}"],
|
||||
capture_output=True, text=True, check=True)
|
||||
cp = r.stdout.strip()
|
||||
return container_to_host(cp) if cp else None
|
||||
|
||||
|
||||
def strip_genre(filepath):
|
||||
"""Blank GENRE so spotify-genre.py will refill it (skip-existing logic)."""
|
||||
m = MFile(filepath)
|
||||
if m is None or m.tags is None:
|
||||
return
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
if "genre" in m.tags:
|
||||
del m.tags["genre"]
|
||||
m.save()
|
||||
elif isinstance(m, MP4):
|
||||
if "\xa9gen" in m.tags:
|
||||
del m.tags["\xa9gen"]
|
||||
m.save()
|
||||
else:
|
||||
try:
|
||||
t = ID3(filepath)
|
||||
t.delall("TCON")
|
||||
t.save(filepath)
|
||||
except ID3NoHeaderError:
|
||||
pass
|
||||
|
||||
|
||||
def refill_genre(beets_id):
|
||||
subprocess.run(
|
||||
[SPOTIFY_GENRE, "--apply", "--force", "--query", f"id:{beets_id}"],
|
||||
check=False)
|
||||
|
||||
|
||||
def write_cover_sidecar(filepath, cover_bytes):
|
||||
if not cover_bytes:
|
||||
return
|
||||
sidecar = os.path.join(os.path.dirname(filepath), "cover.jpg")
|
||||
with open(sidecar, "wb") as f:
|
||||
f.write(cover_bytes)
|
||||
|
||||
|
||||
def navidrome_scan():
|
||||
qs = urllib.parse.urlencode({
|
||||
"u": NAVIDROME_USER, "p": NAVIDROME_PASS,
|
||||
"v": "1.16.0", "c": "fix-track-metadata", "f": "json",
|
||||
})
|
||||
try:
|
||||
urllib.request.urlopen(f"{NAVIDROME_URL}?{qs}", timeout=10).read()
|
||||
print("[navidrome] rescan triggered")
|
||||
except Exception as e:
|
||||
print(f"[navidrome] rescan failed: {e}")
|
||||
|
||||
|
||||
# ---------- Main ----------
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("file", nargs="?",
|
||||
help="audio file: host path, /music/... container path, "
|
||||
"or library-relative path (Foo/Bar.flac)")
|
||||
ap.add_argument("url", nargs="?", help="Spotify or iTunes URL")
|
||||
ap.add_argument("--keep-genre", action="store_true",
|
||||
help="don't strip + refill GENRE (default: refill via spotify-genre.py)")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Tab-completion on the file prompt — pasting unicode paths is finicky,
|
||||
# tab-completing them is not.
|
||||
try:
|
||||
import readline
|
||||
readline.set_completer_delims(" \t\n;")
|
||||
def _path_complete(text, state):
|
||||
base = text if text.startswith("/") else os.path.join(LIBRARY, text)
|
||||
d, prefix = os.path.split(base)
|
||||
if not os.path.isdir(d):
|
||||
return None
|
||||
matches = [n for n in os.listdir(d) if n.startswith(prefix)]
|
||||
matches.sort()
|
||||
if state >= len(matches):
|
||||
return None
|
||||
full = os.path.join(d, matches[state])
|
||||
# Strip the LIBRARY prefix back off if user was typing relative
|
||||
display = full[len(LIBRARY) + 1:] if (
|
||||
not text.startswith("/") and full.startswith(LIBRARY + "/")) else full
|
||||
return display + ("/" if os.path.isdir(full) else "")
|
||||
readline.set_completer(_path_complete)
|
||||
readline.parse_and_bind("tab: complete")
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
raw_file = args.file or input("file path: ").strip()
|
||||
url = args.url or input("spotify/itunes URL: ").strip()
|
||||
filepath = resolve_input_path(raw_file)
|
||||
|
||||
if not os.path.isfile(filepath):
|
||||
# Diagnostics: show the bytes we tried + what's actually in the parent.
|
||||
print(f"[error] not a file: {filepath}", file=sys.stderr)
|
||||
print(f"[error] input bytes: {raw_file.encode('utf-8')!r}", file=sys.stderr)
|
||||
print(f"[error] resolved bytes: {filepath.encode('utf-8')!r}", file=sys.stderr)
|
||||
parent = os.path.dirname(filepath)
|
||||
if os.path.isdir(parent):
|
||||
print(f"[error] parent dir contents:", file=sys.stderr)
|
||||
for n in sorted(os.listdir(parent)):
|
||||
print(f" {n!r} ({n.encode('utf-8')!r})", file=sys.stderr)
|
||||
else:
|
||||
# Walk up to the deepest existing ancestor and list it
|
||||
anc = parent
|
||||
while anc and anc != "/" and not os.path.isdir(anc):
|
||||
anc = os.path.dirname(anc)
|
||||
if anc and os.path.isdir(anc):
|
||||
print(f"[error] deepest existing dir: {anc}", file=sys.stderr)
|
||||
print(f"[error] its contents:", file=sys.stderr)
|
||||
for n in sorted(os.listdir(anc)):
|
||||
print(f" {n!r}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print(f"[fetch] {url}")
|
||||
meta = fetch_metadata(url)
|
||||
print(f" title: {meta['title']}")
|
||||
print(f" artists: {'; '.join(meta['artists'])}")
|
||||
print(f" album: {meta['album']}")
|
||||
print(f" albumartist: {meta['albumartists'][0] if meta['albumartists'] else '?'}")
|
||||
print(f" date: {meta['date']}")
|
||||
print(f" track: {meta['track']}/{meta['tracktotal']}")
|
||||
print(f" cover: {meta['cover_url'] or '(none)'}")
|
||||
|
||||
if not args.file:
|
||||
ok = input("apply? [y/N] ").strip().lower()
|
||||
if ok != "y":
|
||||
print("aborted")
|
||||
return
|
||||
|
||||
print("[tags] writing")
|
||||
cover_bytes = _download_cover(meta["cover_url"]) if meta["cover_url"] else None
|
||||
write_tags(filepath, meta)
|
||||
write_cover_sidecar(filepath, cover_bytes)
|
||||
|
||||
bid = beets_id_for(filepath)
|
||||
if bid is None:
|
||||
print("[beets] file not in beets DB — skipping update/move/genre")
|
||||
else:
|
||||
print(f"[beets] update id:{bid}")
|
||||
beet_update(bid)
|
||||
print(f"[beets] move id:{bid}")
|
||||
beet_move(bid)
|
||||
new_path = beet_path(bid)
|
||||
if new_path:
|
||||
filepath = new_path
|
||||
# Re-drop sidecar in the (possibly new) folder
|
||||
write_cover_sidecar(filepath, cover_bytes)
|
||||
print(f"[beets] now at {filepath}")
|
||||
|
||||
if not args.keep_genre:
|
||||
print("[genre] stripping + refilling via spotify-genre.py")
|
||||
strip_genre(filepath)
|
||||
beet_update(bid)
|
||||
refill_genre(bid)
|
||||
beet_update(bid)
|
||||
|
||||
navidrome_scan()
|
||||
print("[done]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/bash
|
||||
# gen-djmix-playlist.sh — regenerate djmix.m3u8 from the configured album list.
|
||||
#
|
||||
# Reads ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/djmix-albums.txt (one beets query per line),
|
||||
# collects all matching track paths, deduplicates, and writes
|
||||
# ${MUSIC_DATA_DIR:-/data/music}/playlists/djmix.m3u8.
|
||||
#
|
||||
# Azuracast imports this playlist so DJ mixes can be excluded from general
|
||||
# rotation. Navidrome also picks it up as a regular playlist.
|
||||
#
|
||||
# Runs daily from cron. Safe to run manually anytime.
|
||||
|
||||
set -u
|
||||
|
||||
CONFIG=${ALEMBIC_CONFIG_DIR:-/config}/pipeline/djmix-albums.txt
|
||||
PLAYLIST_DIR=${MUSIC_DATA_DIR:-/data/music}/playlists
|
||||
OUT="${PLAYLIST_DIR}/djmix.m3u8"
|
||||
LOG_DIR=${ALEMBIC_CONFIG_DIR:-/config}/logs
|
||||
LOG="${LOG_DIR}/djmix-$(date +%Y%m%d).log"
|
||||
|
||||
mkdir -p "$LOG_DIR" "$PLAYLIST_DIR"
|
||||
|
||||
log() { echo "[$(date -Iseconds)] $*" | tee -a "$LOG"; }
|
||||
|
||||
log "=== Regenerating djmix.m3u8 ==="
|
||||
|
||||
TRACKS=$(mktemp)
|
||||
|
||||
while IFS= read -r query; do
|
||||
[[ -z "$query" || "$query" == "#"* ]] && continue
|
||||
# Split on | to support multi-field queries (e.g. "albumartist::Foo | album:Bar").
|
||||
# Each |-separated part becomes a separate beet query term (ANDed together).
|
||||
# Single-field lines (no |) are passed as one arg so commas in album names
|
||||
# don't get misinterpreted by beet's query parser.
|
||||
IFS='|' read -ra _terms <<< "$query"
|
||||
_args=()
|
||||
for _t in "${_terms[@]}"; do
|
||||
# strip leading/trailing whitespace
|
||||
_t="${_t#"${_t%%[![:space:]]*}"}"
|
||||
_t="${_t%"${_t##*[![:space:]]}"}"
|
||||
[[ -n "$_t" ]] && _args+=("$_t")
|
||||
done
|
||||
beet ls -f '$path' "${_args[@]}" 2>>"$LOG" >> "$TRACKS"
|
||||
done < "$CONFIG"
|
||||
|
||||
# Deduplicate and sort
|
||||
SORTED=$(sort -u "$TRACKS")
|
||||
rm -f "$TRACKS"
|
||||
|
||||
COUNT=$(echo -n "$SORTED" | grep -c '^' || true)
|
||||
|
||||
if [[ "$COUNT" -gt 0 ]]; then
|
||||
{
|
||||
echo "#EXTM3U"
|
||||
echo "$SORTED"
|
||||
} > "$OUT"
|
||||
log "djmix.m3u8 updated with $COUNT tracks"
|
||||
else
|
||||
log "WARNING: no tracks found — djmix.m3u8 not updated"
|
||||
fi
|
||||
|
||||
log "=== Finished ==="
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
# gen-vgm-playlist.sh — regenerate vgm.m3u8 from all tracks tagged with
|
||||
# genre "Game Soundtrack" in the beets library.
|
||||
#
|
||||
# Navidrome picks this up as a regular playlist. Safe to run manually anytime.
|
||||
|
||||
set -u
|
||||
|
||||
PLAYLIST_DIR=${MUSIC_DATA_DIR:-/data/music}/playlists
|
||||
OUT="${PLAYLIST_DIR}/vgm.m3u8"
|
||||
LOG_DIR=${ALEMBIC_CONFIG_DIR:-/config}/logs
|
||||
LOG="${LOG_DIR}/vgm-$(date +%Y%m%d).log"
|
||||
|
||||
mkdir -p "$LOG_DIR" "$PLAYLIST_DIR"
|
||||
|
||||
log() { echo "[$(date -Iseconds)] $*" | tee -a "$LOG"; }
|
||||
|
||||
log "=== Regenerating vgm.m3u8 ==="
|
||||
|
||||
SORTED=$(beet ls -f '$path' 'genres:Game' 2>>"$LOG" | sort -u)
|
||||
COUNT=$(echo -n "$SORTED" | grep -c '^' || true)
|
||||
|
||||
if [[ "$COUNT" -gt 0 ]]; then
|
||||
{
|
||||
echo "#EXTM3U"
|
||||
echo "$SORTED"
|
||||
} > "$OUT"
|
||||
log "vgm.m3u8 updated with $COUNT tracks"
|
||||
else
|
||||
log "WARNING: no tracks found — vgm.m3u8 not updated"
|
||||
fi
|
||||
|
||||
log "=== Finished ==="
|
||||
Executable
+1000
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
||||
#!/bin/bash
|
||||
# ${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
#
|
||||
# Shared helper exposing `strip_mb_tags <dir> [log_file]`.
|
||||
# Sourced by lib/strip-mb-tags.sh (Sunday library-wide scan) and by
|
||||
# import-track.sh (per-import scoped scan).
|
||||
#
|
||||
# Strips MusicBrainz IDs and release-identification tags from FLAC files
|
||||
# that carry at least one trigger tag. Navidrome's BFR scanner uses these
|
||||
# tags to differentiate releases of the same album; when one track in an
|
||||
# album has them and the rest don't, the album card splits.
|
||||
#
|
||||
# Usage: strip_mb_tags <dir> [log_file]
|
||||
# Returns: number of files stripped, via stdout
|
||||
|
||||
_MB_STRIP_TAGS=(
|
||||
MUSICBRAINZ_ALBUMID
|
||||
MUSICBRAINZ_RELEASEGROUPID
|
||||
MUSICBRAINZ_RELEASETRACKID
|
||||
MUSICBRAINZ_ALBUMARTISTID
|
||||
MUSICBRAINZ_ARTISTID
|
||||
MUSICBRAINZ_ALBUMSTATUS
|
||||
MUSICBRAINZ_ALBUMTYPE
|
||||
MUSICBRAINZ_TRACKID
|
||||
MUSICBRAINZ_WORKID
|
||||
RELEASESTATUS
|
||||
RELEASETYPE
|
||||
RELEASECOUNTRY
|
||||
BARCODE
|
||||
CATALOGNUMBER
|
||||
LABEL
|
||||
PUBLISHER
|
||||
MEDIA
|
||||
ORIGINALDATE
|
||||
ASIN
|
||||
ISRC
|
||||
SCRIPT
|
||||
LANGUAGE
|
||||
MUSICBRAINZ_ALBUMCOMMENT
|
||||
ALBUMCOMMENT
|
||||
MUSICBRAINZ_DISCID
|
||||
MUSICBRAINZ_TRMID
|
||||
"ALBUM ARTIST"
|
||||
ALBUM_ARTIST
|
||||
ALBUMARTIST_CREDIT
|
||||
ALBUMARTISTSORT
|
||||
ALBUMARTISTS
|
||||
ALBUM_ARTISTS
|
||||
ALBUMARTISTS_CREDIT
|
||||
ALBUMARTISTS_SORT
|
||||
ARTIST_CREDIT
|
||||
ARTISTSORT
|
||||
ARTISTS
|
||||
ARTISTS_CREDIT
|
||||
ARTISTS_SORT
|
||||
COMPOSERSORT
|
||||
ACOUSTID_ID
|
||||
ACOUSTID_FINGERPRINT
|
||||
COMPILATION
|
||||
YEAR
|
||||
DISCSUBTITLE
|
||||
DISCC
|
||||
TOTALDISCS
|
||||
TRACKC
|
||||
TOTALTRACKS
|
||||
)
|
||||
|
||||
_MB_STRIP_TRIGGERS=(
|
||||
MUSICBRAINZ_ALBUMID
|
||||
MUSICBRAINZ_RELEASEGROUPID
|
||||
MUSICBRAINZ_TRACKID
|
||||
BARCODE
|
||||
CATALOGNUMBER
|
||||
LABEL
|
||||
ORIGINALDATE
|
||||
ASIN
|
||||
MUSICBRAINZ_ALBUMCOMMENT
|
||||
ALBUMCOMMENT
|
||||
)
|
||||
|
||||
strip_mb_tags() {
|
||||
local dir="$1"
|
||||
local logfile="${2:-/dev/null}"
|
||||
local count=0
|
||||
local f tt has_trigger args
|
||||
|
||||
if [[ -z "$dir" || ! -e "$dir" ]]; then
|
||||
echo 0
|
||||
return 0
|
||||
fi
|
||||
|
||||
while IFS= read -r -d '' f; do
|
||||
has_trigger=0
|
||||
for tt in "${_MB_STRIP_TRIGGERS[@]}"; do
|
||||
if metaflac --show-tag="$tt" "$f" 2>/dev/null | grep -q .; then
|
||||
has_trigger=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ $has_trigger -eq 1 ]]; then
|
||||
args=()
|
||||
for t in "${_MB_STRIP_TAGS[@]}"; do args+=("--remove-tag=$t"); done
|
||||
metaflac "${args[@]}" "$f" 2>>"$logfile"
|
||||
count=$((count + 1))
|
||||
fi
|
||||
done < <(find "$dir" -name "*.flac" -type f -print0 2>/dev/null)
|
||||
|
||||
echo "$count"
|
||||
}
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# navidrome-scan.sh — trigger a Navidrome full scan ONLY if the music library
|
||||
# share is healthy.
|
||||
#
|
||||
# Why this exists: with ND_SCANNER_PURGEMISSING=always (which we keep, so
|
||||
# on-disk deletions propagate to playlists and out to Traktor), a full scan
|
||||
# that runs while the QNAP NFS share is unavailable makes Navidrome see the
|
||||
# entire library as "missing" and purge it — wiping every playlist. That is
|
||||
# exactly what happened on 2026-06-01 and lost the manual dj-* playlists.
|
||||
#
|
||||
# This wrapper refuses to trigger the scan unless the share is mounted, a
|
||||
# canary file is readable, and the library has a sane number of artist dirs.
|
||||
# Skipping a scan is harmless (the next healthy run picks up changes); running
|
||||
# one against a dead mount is catastrophic.
|
||||
set -euo pipefail
|
||||
|
||||
: "${MUSIC_DATA_DIR:=/data/music}"
|
||||
: "${ALEMBIC_CONFIG_DIR:=/config}"
|
||||
NAVIDROME_ENV="$ALEMBIC_CONFIG_DIR/pipeline/navidrome/admin.env"
|
||||
[ -f "$NAVIDROME_ENV" ] && source "$NAVIDROME_ENV"
|
||||
|
||||
LIB="$MUSIC_DATA_DIR/Library"
|
||||
CANARY="$LIB/.navidrome-canary"
|
||||
MIN_ARTIST_DIRS=500 # library has ~1500; 500 is "clearly not wiped"
|
||||
ND="${ND_BASE:-http://navidrome:4533}"
|
||||
ND_USER="${ND_USER:-andrew}"
|
||||
ND_PASS="${ND_PASS:-}"
|
||||
|
||||
log() { echo "[$(date -Iseconds)] navidrome-scan: $*"; }
|
||||
|
||||
# `mountpoint -q` on the host detected an unmounted/dead NFS share directly.
|
||||
# Running inside a container against a bind-mounted view, that check doesn't
|
||||
# apply (a bind mount has no distinct device number to detect) — the canary
|
||||
# file + artist-dir-count checks below are what actually carry the
|
||||
# 2026-06-01 protection now: an unhealthy/empty share fails both of them.
|
||||
# (Confirmed as an intentional adaptation, not a silent drop — see alembic
|
||||
# migration plan / MIGRATION.md.)
|
||||
if [ ! -r "$CANARY" ]; then
|
||||
log "ABORT: canary $CANARY missing/unreadable — share degraded — skipping scan"
|
||||
exit 0
|
||||
fi
|
||||
n=$(find "$LIB" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l)
|
||||
if [ "$n" -lt "$MIN_ARTIST_DIRS" ]; then
|
||||
log "ABORT: only $n artist dirs under $LIB (< $MIN_ARTIST_DIRS) — library looks truncated — skipping scan"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "library healthy ($n artist dirs) — triggering full scan"
|
||||
resp=$(curl -s -G "$ND/rest/startScan.view" \
|
||||
--data-urlencode "u=$ND_USER" --data-urlencode "p=$ND_PASS" \
|
||||
--data-urlencode 'v=1.16.0' --data-urlencode 'c=cron' \
|
||||
--data-urlencode 'f=json' --data-urlencode 'fullScan=true' || true)
|
||||
# Don't log "scan triggered" on faith: a rotated password or Navidrome error
|
||||
# would otherwise report success forever while no scan ever runs.
|
||||
if echo "$resp" | grep -q '"status":"ok"'; then
|
||||
log "scan triggered"
|
||||
else
|
||||
log "ERROR: scan trigger failed — response: ${resp:-<empty/no response>}"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+110
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
normalize-artist-casing.py — enforce canonical artist-name casing in beets.
|
||||
|
||||
Reads $ALEMBIC_CONFIG_DIR/pipeline/artist-canonical.list (one canonical name per line,
|
||||
`#` comments allowed). For each distinct $albumartist in the library that
|
||||
matches a canonical name case-insensitively but differs in casing, runs
|
||||
`beet modify` to rewrite the albumartist tag — which moves the album folder
|
||||
to the canonical path as a side effect.
|
||||
|
||||
Idempotent. Cheap to run (one `beet ls` + one `beet modify` per mismatch).
|
||||
|
||||
Why: prevents Linux → Windows Syncthing case-conflicts. Linux treats
|
||||
`Mall Grab/` and `MALL GRAB/` as distinct folders; Windows collapses them.
|
||||
|
||||
Usage:
|
||||
normalize-artist-casing.py # dry-run, prints diffs
|
||||
normalize-artist-casing.py --apply # do the work
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
CONFIG = f"{os.environ.get('ALEMBIC_CONFIG_DIR', '/config')}/pipeline/artist-canonical.list"
|
||||
BEET = ["beet"]
|
||||
|
||||
|
||||
def load_canonical(path: str) -> list[str]:
|
||||
out: list[str] = []
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for ln in f:
|
||||
ln = ln.split("#", 1)[0].strip()
|
||||
if ln:
|
||||
out.append(ln)
|
||||
return out
|
||||
|
||||
|
||||
def all_albumartists() -> set[str]:
|
||||
res = subprocess.run(BEET + ["ls", "-f", "$albumartist"],
|
||||
capture_output=True, text=True, check=True)
|
||||
return {ln.strip() for ln in res.stdout.splitlines() if ln.strip()}
|
||||
|
||||
|
||||
def apply_fix(old: str, new: str, verbose: bool) -> bool:
|
||||
query = f"albumartist::^{re.escape(old)}$"
|
||||
cmd = BEET + ["modify", "-y", query, f"albumartist={new}"]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if verbose:
|
||||
sys.stdout.write(r.stdout)
|
||||
if r.returncode != 0:
|
||||
sys.stderr.write(f" ERROR ({old!r} → {new!r}): {r.stderr.strip()}\n")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--apply", action="store_true",
|
||||
help="execute the renames (default: dry run)")
|
||||
ap.add_argument("--verbose", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
stamp = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
canonical = load_canonical(CONFIG)
|
||||
by_lower: dict[str, str] = {}
|
||||
for n in canonical:
|
||||
key = n.lower()
|
||||
if key in by_lower and by_lower[key] != n:
|
||||
print(f"[{stamp}] WARN: {CONFIG} has two entries with same lowercased "
|
||||
f"key: {by_lower[key]!r} and {n!r}; using the latter",
|
||||
file=sys.stderr)
|
||||
by_lower[key] = n
|
||||
|
||||
existing = all_albumartists()
|
||||
|
||||
mismatches: list[tuple[str, str]] = []
|
||||
for aa in sorted(existing):
|
||||
want = by_lower.get(aa.lower())
|
||||
if want and aa != want:
|
||||
mismatches.append((aa, want))
|
||||
|
||||
print(f"[{stamp}] normalize-artist-casing: {len(canonical)} canonical, "
|
||||
f"{len(existing)} distinct album-artists in library, "
|
||||
f"{len(mismatches)} mismatched")
|
||||
|
||||
if not mismatches:
|
||||
return 0
|
||||
|
||||
for old, new in mismatches:
|
||||
print(f" {old!r} -> {new!r}")
|
||||
|
||||
if not args.apply:
|
||||
print(f"[{stamp}] dry run; pass --apply to execute")
|
||||
return 0
|
||||
|
||||
fixed = 0
|
||||
for old, new in mismatches:
|
||||
if apply_fix(old, new, args.verbose):
|
||||
fixed += 1
|
||||
print(f"[{stamp}] done: {fixed}/{len(mismatches)} applied")
|
||||
return 0 if fixed == len(mismatches) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
# notify-telegram.sh — send a message to the configured Telegram chat.
|
||||
# Sources credentials from $ALEMBIC_CONFIG_DIR/pipeline/telegram/notify.env (chmod 600).
|
||||
#
|
||||
# Usage:
|
||||
# echo "single line message" | notify-telegram.sh
|
||||
# notify-telegram.sh "single line message"
|
||||
# notify-telegram.sh < ${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log
|
||||
#
|
||||
# Telegram messages are capped at 4096 chars. Anything longer is truncated
|
||||
# with a "...<truncated>" tail. Returns exit 0 on send-OK, non-zero otherwise.
|
||||
|
||||
set -u
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
CONFIG="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/telegram/notify.env"
|
||||
if [[ ! -f "$CONFIG" ]]; then
|
||||
echo "[notify-telegram] no config at $CONFIG" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG"
|
||||
: "${TG_BOT_TOKEN:?TG_BOT_TOKEN missing}"
|
||||
: "${TG_CHAT_ID:?TG_CHAT_ID missing}"
|
||||
|
||||
# Read message from arg or stdin
|
||||
if [[ $# -ge 1 ]]; then
|
||||
MSG="$*"
|
||||
else
|
||||
MSG=$(cat)
|
||||
fi
|
||||
|
||||
# Telegram caps at 4096 chars (UTF-8 codepoints). Trim conservatively at 3900.
|
||||
# Use python for proper UTF-8 length handling.
|
||||
MSG=$(python3 -c "
|
||||
import sys
|
||||
s = sys.argv[1]
|
||||
if len(s) > 3900:
|
||||
s = s[:3900] + '\n...<truncated>'
|
||||
print(s, end='')
|
||||
" "$MSG")
|
||||
|
||||
# Send via Bot API. Disable web-page preview and use plain text (no parse_mode)
|
||||
# so log content with special chars doesn't get interpreted as Markdown.
|
||||
resp=$(curl -s --max-time 15 -X POST \
|
||||
"https://api.telegram.org/bot${TG_BOT_TOKEN}/sendMessage" \
|
||||
--data-urlencode "chat_id=${TG_CHAT_ID}" \
|
||||
--data-urlencode "text=${MSG}" \
|
||||
--data-urlencode "disable_web_page_preview=true")
|
||||
|
||||
ok=$(echo "$resp" | python3 -c "import sys,json;print(json.load(sys.stdin).get('ok',False))" 2>/dev/null || echo False)
|
||||
if [[ "$ok" != "True" ]]; then
|
||||
echo "[notify-telegram] send failed: $resp" >&2
|
||||
exit 2
|
||||
fi
|
||||
exit 0
|
||||
Executable
+390
@@ -0,0 +1,390 @@
|
||||
#!/bin/bash
|
||||
# pipeline-status.sh — daily health check for the music pipeline.
|
||||
#
|
||||
# Writes a one-screen summary to ${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log (overwritten daily)
|
||||
# and ships it to Telegram via notify-telegram.sh.
|
||||
#
|
||||
# Reports on:
|
||||
# - Sibling service reachability (slskd, navidrome) via HTTP, not docker ps —
|
||||
# alembic has no Docker socket access
|
||||
# - Today's runs: playlist syncs, Bandcamp sync, manual imports, dedup
|
||||
# - Weekly maintenance freshness: strip-mb-tags, strip-watermark-art,
|
||||
# scrub-watermark-text, clean-sldl-index, spotify-genre
|
||||
# - Daily dedup freshness
|
||||
# - Monthly mp3→flac upgrade freshness
|
||||
# - Auth & state: Bandcamp cookie expiry, Qobuz token liveness, VPN egress
|
||||
# - Storage, Navidrome scan freshness, library size by format
|
||||
#
|
||||
# Only flags SCRIPT-level failures, not per-file Soulseek errors (those are
|
||||
# normal — peers go offline, connections reset).
|
||||
|
||||
set -u
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
OUT=${ALEMBIC_CONFIG_DIR:-/config}/logs/STATUS.log
|
||||
TODAY=$(date +%Y%m%d)
|
||||
NOW_TS=$(date +%s)
|
||||
|
||||
NAVIDROME_ENV="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/navidrome/admin.env"
|
||||
[ -f "$NAVIDROME_ENV" ] && source "$NAVIDROME_ENV"
|
||||
ND_BASE="${ND_BASE:-http://navidrome:4533}"
|
||||
ND_USER="${ND_USER:-andrew}"
|
||||
ND_PASS="${ND_PASS:-}"
|
||||
|
||||
# Tally counters for the header banner. Body collected first, then prepended.
|
||||
OK=0
|
||||
WARN=0
|
||||
SKIP=0
|
||||
|
||||
mark_ok() { OK=$((OK+1)); printf " ✓ %s\n" "$*"; }
|
||||
mark_warn() { WARN=$((WARN+1)); printf " ⚠ %s\n" "$*"; }
|
||||
mark_skip() { SKIP=$((SKIP+1)); printf " ⊘ %s\n" "$*"; }
|
||||
mark_info() { printf " %s\n" "$*"; }
|
||||
|
||||
section() { printf "\n▎ %s\n" "$*"; }
|
||||
|
||||
# Format an age in seconds as "Nh", "Nd", "Nw", "Nmo".
|
||||
human_age() {
|
||||
local s=$1
|
||||
if (( s < 3600 )); then echo "$((s/60))m ago"
|
||||
elif (( s < 86400 )); then echo "$((s/3600))h ago"
|
||||
elif (( s < 86400*14 )); then echo "$((s/86400))d ago"
|
||||
elif (( s < 86400*60 )); then echo "$((s/86400/7))w ago"
|
||||
else echo "$((s/86400/30))mo ago"
|
||||
fi
|
||||
}
|
||||
|
||||
# Newest matching log → "<age_seconds>|<path>", empty if no match.
|
||||
newest_log() {
|
||||
local glob="$1"
|
||||
local newest
|
||||
newest=$(ls -1t $glob 2>/dev/null | head -1)
|
||||
[[ -z "$newest" ]] && return
|
||||
local mtime
|
||||
mtime=$(stat -c %Y "$newest" 2>/dev/null) || return
|
||||
echo "$((NOW_TS - mtime))|$newest"
|
||||
}
|
||||
|
||||
# Per-playlist log status.
|
||||
playlist_status() {
|
||||
local log="$1"
|
||||
if grep -q "=== Finished playlist run" "$log" 2>/dev/null; then
|
||||
# awk picks just the count after "with"; sidesteps the "M3U" digit-3 trap
|
||||
# that bites grep -oE '[0-9]+'.
|
||||
local m3u
|
||||
m3u=$(awk 'match($0, /updated with [0-9]+ tracks/) {
|
||||
n=substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", n); last=n
|
||||
} END { print last }' "$log")
|
||||
echo "OK|${m3u:-0} tracks in M3U"
|
||||
elif grep -q "=== Starting playlist run" "$log" 2>/dev/null; then
|
||||
echo "WARN|started but never finished"
|
||||
else
|
||||
echo "WARN|no Starting line"
|
||||
fi
|
||||
}
|
||||
|
||||
bandcamp_status() {
|
||||
local log="$1"
|
||||
if grep -q "=== Bandcamp sync done" "$log" 2>/dev/null; then
|
||||
local s
|
||||
s=$(grep -oE "\[bandcamp\] === [0-9]+ new[^]]*" "$log" | tail -1 | sed 's/\[bandcamp\] === //; s/ ===$//')
|
||||
echo "OK|${s:-completed}"
|
||||
else
|
||||
echo "WARN|started but never finished"
|
||||
fi
|
||||
}
|
||||
|
||||
manual_import_status() {
|
||||
local log="$1"
|
||||
local n
|
||||
# || true, not || echo 0: grep -c already prints "0" on no-match (and exits
|
||||
# 1), so || echo 0 produced a two-line "0\n0" that broke the -ge test.
|
||||
n=$(grep -c "=== Finished manual import" "$log" 2>/dev/null || true)
|
||||
if [[ "${n:-0}" -ge 1 ]]; then
|
||||
echo "OK|$n run(s)"
|
||||
else
|
||||
echo "WARN|started but never finished"
|
||||
fi
|
||||
}
|
||||
|
||||
dedup_today_status() {
|
||||
local log="$1"
|
||||
if grep -q "=== Summary:" "$log" 2>/dev/null; then
|
||||
# Format: "=== Summary: N dup groups (parens with commas), N kept, N to delete ==="
|
||||
# Strip the wrapper and the parenthesized breakdown so the line stays short.
|
||||
local s
|
||||
s=$(grep "=== Summary:" "$log" | tail -1 \
|
||||
| sed -E 's/.*=== Summary: //; s/ ===.*//; s/ \([^)]*\)//')
|
||||
echo "OK|${s:-completed (no summary line)}"
|
||||
else
|
||||
echo "WARN|started but never finished"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Build report ----
|
||||
{
|
||||
echo "__HEADER_PLACEHOLDER__"
|
||||
|
||||
# 1. Sibling service health
|
||||
# alembic deliberately has no Docker socket access (nothing left needs it
|
||||
# once sldl/beet run in-process), so these are HTTP reachability checks
|
||||
# against each service's own endpoint rather than `docker ps` container
|
||||
# state — arguably a better signal anyway (checks the service actually
|
||||
# answers, not just that the container process exists).
|
||||
section "Services"
|
||||
check_http() {
|
||||
local name="$1" url="$2"
|
||||
if curl -s -o /dev/null --connect-timeout 3 --max-time 6 "$url"; then
|
||||
mark_ok "$(printf '%-11s reachable' "$name")"
|
||||
else
|
||||
mark_warn "$(printf '%-11s unreachable at %s' "$name" "$url")"
|
||||
fi
|
||||
}
|
||||
check_http slskd "http://gluetun:5030/"
|
||||
check_http navidrome "http://navidrome:4533/rest/ping.view"
|
||||
|
||||
# 2. Today's runs
|
||||
# Allowlist-based: only Spotify playlists, Bandcamp, manual import, and
|
||||
# dedup count as "runs." Everything else (laptop-export, normalize-casing,
|
||||
# gen-djmix, gen-vgm, fix-empty-album, fingerprint-index, backfill, replace,
|
||||
# convert-m4a, fred-mixes, dj-import-*, weekly tag-hygiene) is either
|
||||
# boring/silent maintenance or already covered in the Maintenance section.
|
||||
section "Today's runs"
|
||||
any_today=0
|
||||
newest_dedup_today=$(ls -1t ${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$TODAY-*.log 2>/dev/null | head -1)
|
||||
|
||||
# Known Spotify playlist names — derive from configs/ directory so adding a
|
||||
# new playlist auto-includes it here without editing this script.
|
||||
PLAYLIST_NAMES=$(ls ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/*.conf 2>/dev/null \
|
||||
| xargs -n1 basename 2>/dev/null \
|
||||
| sed 's/\.conf$//' \
|
||||
| grep -v '^_' || true)
|
||||
|
||||
for log in ${ALEMBIC_CONFIG_DIR:-/config}/logs/*-$TODAY.log; do
|
||||
[[ -f "$log" ]] || continue
|
||||
name=$(basename "$log" .log)
|
||||
short=${name%-$TODAY}
|
||||
case "$short" in
|
||||
manual-import)
|
||||
any_today=1
|
||||
IFS='|' read -r tag detail < <(manual_import_status "$log") ;;
|
||||
bandcamp)
|
||||
any_today=1
|
||||
IFS='|' read -r tag detail < <(bandcamp_status "$log") ;;
|
||||
*)
|
||||
# Is this a Spotify playlist log? Check against PLAYLIST_NAMES.
|
||||
if echo "$PLAYLIST_NAMES" | grep -qxF "$short"; then
|
||||
any_today=1
|
||||
IFS='|' read -r tag detail < <(playlist_status "$log")
|
||||
else
|
||||
continue # not a "run" — silent / belongs in Maintenance / irrelevant
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
line=$(printf '%-13s %s' "$short" "$detail")
|
||||
if [[ "$tag" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
||||
done
|
||||
if [[ -n "$newest_dedup_today" ]]; then
|
||||
any_today=1
|
||||
IFS='|' read -r tag detail < <(dedup_today_status "$newest_dedup_today")
|
||||
line=$(printf '%-13s %s' "dedup" "$detail")
|
||||
if [[ "$tag" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
||||
fi
|
||||
[[ "$any_today" -eq 0 ]] && mark_info "(nothing has run yet today)"
|
||||
|
||||
# 2b. Nightly download tally — count tracks added to beets in the last 24h.
|
||||
# Captures the overnight playlist sweep + any daytime Bandcamp/manual imports.
|
||||
# Uses beets' `added` timestamp, so it's accurate regardless of how files
|
||||
# arrived (Spotify pipeline, Bandcamp, manual). Per-playlist breakdown
|
||||
# follows when ≥1 track was added with that grouping today.
|
||||
ADDED_TODAY=$(beet ls -f '$grouping' \
|
||||
"added:$(date +%Y-%m-%d).." 2>/dev/null || true)
|
||||
added_total=$(echo -n "$ADDED_TODAY" | grep -c '^' || true)
|
||||
mark_ok "$(printf '%-13s %d new tracks in beets' 'added today' "$added_total")"
|
||||
if [[ "$added_total" -gt 0 ]]; then
|
||||
# Per-playlist breakdown. Empty $grouping (bandcamp/manual) shown as "(none)".
|
||||
echo "$ADDED_TODAY" \
|
||||
| awk '{ if ($0 == "") print "(none)"; else print }' \
|
||||
| sort | uniq -c | sort -rn \
|
||||
| awk '{ g=$2; for(i=3;i<=NF;i++) g=g" "$i; printf " %3d %s\n", $1, g }'
|
||||
fi
|
||||
|
||||
# 3. Maintenance freshness
|
||||
# Each weekly/monthly task: find its newest log and check age.
|
||||
# Limits: weekly tasks should be <9 days old; daily <2; monthly <35.
|
||||
section "Maintenance (last run)"
|
||||
|
||||
declare -A MAINT_GLOB=(
|
||||
[strip-mb-tags]="${ALEMBIC_CONFIG_DIR:-/config}/logs/mb-strip-*.log"
|
||||
[strip-watermark-art]="${ALEMBIC_CONFIG_DIR:-/config}/logs/strip-watermark-*.log"
|
||||
[scrub-watermark-text]="${ALEMBIC_CONFIG_DIR:-/config}/logs/scrub-text-*.log"
|
||||
[clean-sldl-index]="${ALEMBIC_CONFIG_DIR:-/config}/logs/clean-index-*.log"
|
||||
[clear-bad-genres]="${ALEMBIC_CONFIG_DIR:-/config}/logs/clear-bad-genres-*.log"
|
||||
[spotify-genre]="${ALEMBIC_CONFIG_DIR:-/config}/logs/spotify-genre-*.log"
|
||||
[dedup-library]="${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-*.log"
|
||||
[upgrade-mp3-to-flac]="${ALEMBIC_CONFIG_DIR:-/config}/logs/upgrade-mp3-*.log"
|
||||
)
|
||||
declare -A MAINT_LIMIT=(
|
||||
[strip-mb-tags]=$((9*86400))
|
||||
[strip-watermark-art]=$((9*86400))
|
||||
[scrub-watermark-text]=$((9*86400))
|
||||
[clean-sldl-index]=$((9*86400))
|
||||
[clear-bad-genres]=$((9*86400))
|
||||
[spotify-genre]=$((9*86400))
|
||||
[dedup-library]=$((2*86400))
|
||||
[upgrade-mp3-to-flac]=$((35*86400))
|
||||
)
|
||||
for task in strip-mb-tags strip-watermark-art scrub-watermark-text clean-sldl-index clear-bad-genres spotify-genre dedup-library upgrade-mp3-to-flac; do
|
||||
res=$(newest_log "${MAINT_GLOB[$task]}")
|
||||
if [[ -z "$res" ]]; then
|
||||
mark_skip "$(printf '%-22s never run yet' "$task")"
|
||||
continue
|
||||
fi
|
||||
age_s=${res%%|*}
|
||||
log=${res##*|}
|
||||
age_h=$(human_age "$age_s")
|
||||
case "$task" in
|
||||
spotify-genre)
|
||||
snip=$(grep -oE "written: [0-9]+, unchanged: [0-9]+" "$log" | tail -1) ;;
|
||||
clear-bad-genres)
|
||||
snip=$(grep -oE "(would-clear|cleared): [0-9]+ tracks" "$log" | tail -1) ;;
|
||||
clean-sldl-index)
|
||||
snip=$(grep -oE "[0-9]+ m3us, total [0-9]+ kept, [0-9]+ dropped" "$log" | tail -1) ;;
|
||||
strip-watermark-art)
|
||||
snip=$(grep -oE "stripped from [0-9]+ tracks|no images shared" "$log" | tail -1) ;;
|
||||
scrub-watermark-text)
|
||||
snip=$(grep -oE "stripped [0-9]+ frame\(s\) across [0-9]+ file" "$log" | tail -1) ;;
|
||||
strip-mb-tags)
|
||||
snip=$(grep -oE "Done\. Log:" "$log" | head -1 | sed 's/Done\. Log:/done/') ;;
|
||||
dedup-library)
|
||||
snip=$(grep "=== Summary:" "$log" | tail -1 \
|
||||
| sed -E 's/.*=== Summary: //; s/ ===.*//; s/ \([^)]*\)//') ;;
|
||||
upgrade-mp3-to-flac)
|
||||
# The log's "to attempt" count is frozen at the last monthly run, so on
|
||||
# its own it looks stale against the live "Library by format" section.
|
||||
# Pair it with the current format:mp3 count so the drop (upgrades that
|
||||
# have landed since) is visible instead of looking like a mismatch.
|
||||
attempted=$(grep -oE "[0-9]+ MP3 tracks to attempt" "$log" | grep -oE "^[0-9]+" | tail -1)
|
||||
mp3_now=$(beet ls 'format:mp3' 2>/dev/null | grep -c '^')
|
||||
snip="${attempted:-?} attempted · ${mp3_now} MP3 now" ;;
|
||||
esac
|
||||
line=$(printf '%-22s %-9s %s' "$task" "$age_h" "${snip:-}")
|
||||
if (( age_s > MAINT_LIMIT[$task] )); then
|
||||
mark_warn "$line"
|
||||
else
|
||||
mark_ok "$line"
|
||||
fi
|
||||
done
|
||||
|
||||
# 4. Auth & state
|
||||
section "Auth & state"
|
||||
|
||||
# Bandcamp cookie
|
||||
if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt ]]; then
|
||||
exp=$(awk -F'\t' '$6=="identity" {print $5; exit}' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)
|
||||
if [[ -n "${exp:-}" && "$exp" =~ ^[0-9]+$ && "$exp" -gt 0 ]]; then
|
||||
days=$(( (exp - NOW_TS) / 86400 ))
|
||||
line=$(printf '%-22s %d days left' "Bandcamp cookie" "$days")
|
||||
if (( days < 14 )); then mark_warn "$line — re-export soon"; else mark_ok "$line"; fi
|
||||
else
|
||||
mark_warn "Bandcamp cookie could not parse expiry"
|
||||
fi
|
||||
else
|
||||
mark_warn "Bandcamp cookie missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/cookies.txt)"
|
||||
fi
|
||||
|
||||
# Qobuz Web Player token (buy-link enrich cascade #2). The token is opaque
|
||||
# (no decodable expiry), so probe it live: a cheap authed search returns 200
|
||||
# while valid, 401 once it's logged out — at which point enrich-buy-url.py
|
||||
# silently drops Qobuz and falls through to iTunes.
|
||||
if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token ]]; then
|
||||
q_token=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)
|
||||
q_appid=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/app_id 2>/dev/null)
|
||||
q_appid=${q_appid:-798273057}
|
||||
q_code=$(curl -s -o /dev/null --connect-timeout 5 --max-time 12 -A "Mozilla/5.0" \
|
||||
-H "X-App-Id: $q_appid" -H "X-User-Auth-Token: $q_token" -w '%{http_code}' \
|
||||
"https://www.qobuz.com/api.json/0.2/track/search?query=test&limit=1&app_id=$q_appid" 2>/dev/null)
|
||||
case "$q_code" in
|
||||
200) mark_ok "$(printf '%-22s %s' 'Qobuz token' 'valid (buy-link lookup live)')" ;;
|
||||
401) mark_warn "$(printf '%-22s %s' 'Qobuz token' "EXPIRED — re-export X-User-Auth-Token to ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token")" ;;
|
||||
*) mark_warn "$(printf '%-22s %s' 'Qobuz token' "check failed (HTTP ${q_code:-none})")" ;;
|
||||
esac
|
||||
else
|
||||
mark_warn "$(printf '%-22s %s' 'Qobuz token' "missing (${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)")"
|
||||
fi
|
||||
|
||||
# VPN egress
|
||||
# alembic itself rides gluetun's network namespace (network_mode:
|
||||
# service:gluetun), so its own outbound traffic IS gluetun's traffic —
|
||||
# checking alembic's own egress IP directly is a more direct signal than
|
||||
# asking Docker to introspect gluetun's health, and needs no Docker socket.
|
||||
egress_ip=$(curl -s --connect-timeout 5 --max-time 10 https://ifconfig.me 2>/dev/null)
|
||||
if [[ -n "$egress_ip" ]]; then
|
||||
mark_ok "$(printf '%-22s %s' 'VPN egress' "$egress_ip")"
|
||||
else
|
||||
mark_warn "$(printf '%-22s %s' 'VPN egress' 'could not determine egress IP — VPN may be down')"
|
||||
fi
|
||||
|
||||
# 5. Storage
|
||||
section "Storage"
|
||||
while read -r mp pct used total; do
|
||||
line=$(printf '%-22s %s used (%s of %s)' "$mp" "$pct" "$used" "$total")
|
||||
pct_n=${pct%\%}
|
||||
if (( pct_n > 90 )); then mark_warn "$line"; else mark_ok "$line"; fi
|
||||
done < <(df -h ${MUSIC_DATA_DIR:-/data/music} / 2>/dev/null | awk 'NR>1 && $1!~/tmpfs/ {print $6, $5, $3, $2}')
|
||||
|
||||
# 6. Navidrome
|
||||
section "Navidrome"
|
||||
ND_RESP=$(curl -s --connect-timeout 5 -G "$ND_BASE/rest/getScanStatus.view" \
|
||||
--data-urlencode "u=$ND_USER" \
|
||||
--data-urlencode "p=$ND_PASS" \
|
||||
--data-urlencode 'v=1.16.0' --data-urlencode 'c=status' --data-urlencode 'f=json' 2>/dev/null)
|
||||
ND_LINE=$(echo "$ND_RESP" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
d = json.load(sys.stdin)['subsonic-response']
|
||||
if d.get('status') == 'ok':
|
||||
s = d.get('scanStatus', {})
|
||||
cnt = s.get('count', '?')
|
||||
cnt_str = f'{cnt:,}' if isinstance(cnt, int) else str(cnt)
|
||||
print(f\"OK|last scan {s.get('lastScan','?')[:19]} · {cnt_str} items\")
|
||||
else:
|
||||
print(f\"WARN|status={d.get('status')}\")
|
||||
except Exception as e:
|
||||
print(f\"WARN|unreachable: {e}\")
|
||||
" 2>/dev/null)
|
||||
state=${ND_LINE%%|*}; detail=${ND_LINE##*|}
|
||||
line=$(printf '%-22s %s' "Navidrome" "$detail")
|
||||
if [[ "$state" == "OK" ]]; then mark_ok "$line"; else mark_warn "$line"; fi
|
||||
|
||||
# 7. Library by format (info-only, no status pill)
|
||||
section "Library by format"
|
||||
beet ls -f '$format' 2>/dev/null | sort | uniq -c | sort -rn \
|
||||
| awk '{printf " %-6s %d tracks\n", $2, $1}'
|
||||
|
||||
} > "$OUT"
|
||||
|
||||
# ---- Build header banner and prepend ----
|
||||
HOSTNAME_SHORT=$(hostname -s)
|
||||
DATE_HUMAN=$(TZ=America/Edmonton date '+%a %Y-%m-%d %H:%M %Z')
|
||||
BANNER_TITLE="🎵 Music pipeline · $HOSTNAME_SHORT · $DATE_HUMAN"
|
||||
BANNER_TALLY=" $OK OK · $WARN warn · $SKIP skip"
|
||||
sed -i "1c\\
|
||||
${BANNER_TITLE}\\
|
||||
${BANNER_TALLY}" "$OUT"
|
||||
|
||||
# Always mirror the one-line summary to syslog so journalctl shows last-run
|
||||
# state even if Telegram is broken. Warnings get a separate WARN tag.
|
||||
logger -t sldl-pipeline "status: $OK ok, $WARN warn, $SKIP skip (see $OUT)"
|
||||
if (( WARN > 0 )); then
|
||||
logger -t sldl-pipeline "WARN: pipeline-status flagged $WARN issue(s) — see $OUT"
|
||||
fi
|
||||
|
||||
# Send to Telegram. If it fails, log the failure to syslog so the absence of
|
||||
# a message in the chat has a corresponding journal entry to grep for.
|
||||
if [[ -x ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh ]]; then
|
||||
if ! ${PIPELINE_DIR:-/app/pipeline}/lib/notify-telegram.sh < "$OUT" 2>/tmp/tg-err; then
|
||||
logger -t sldl-pipeline "WARN: Telegram send failed: $(cat /tmp/tg-err 2>/dev/null | head -c 200)"
|
||||
rm -f /tmp/tg-err
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
# ${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
#
|
||||
# Shared helper sourced by run-playlist.sh and import-track.sh.
|
||||
# Pre-tags newly-arrived audio with ReplayGain (loudgain, track-mode) and
|
||||
# Liquidsoap autocue tags (cue_file from Moonbase59/autocue) before beets
|
||||
# ingests them, so AzuraCast sees fully-prepped files the moment they land
|
||||
# in /music.
|
||||
#
|
||||
# Usage: prep_audio <scan_dir> <log_file>
|
||||
# Returns: number of files processed (via stdout)
|
||||
|
||||
prep_audio() {
|
||||
local scan_dir="$1"
|
||||
local log="$2"
|
||||
|
||||
local files=()
|
||||
while IFS= read -r -d '' f; do
|
||||
files+=("$f")
|
||||
done < <(find "$scan_dir" -type f \
|
||||
\( -iname "*.flac" -o -iname "*.mp3" -o -iname "*.m4a" \
|
||||
-o -iname "*.ogg" -o -iname "*.opus" \) -print0)
|
||||
|
||||
if [[ ${#files[@]} -eq 0 ]]; then
|
||||
echo 0
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "[prep-audio] loudgain (track-mode RG, clip-safe) on ${#files[@]} file(s)" >>"$log"
|
||||
# -q quiet, -k lower track gain to avoid clipping >-1 dBFS,
|
||||
# -s e write extended ReplayGain2 + R128 tags, -I 4 ID3v2.4 for MP3.
|
||||
# No -a, so each file gets its own track gain (track-mode).
|
||||
if ! loudgain -q -k -s e -I 4 "${files[@]}" >>"$log" 2>&1; then
|
||||
echo "[prep-audio] loudgain returned non-zero (see log)" >>"$log"
|
||||
fi
|
||||
|
||||
echo "[prep-audio] cue_file (autocue liq_* tags) on ${#files[@]} file(s)" >>"$log"
|
||||
# -w write tags into file, -k true-peak clipping prevention.
|
||||
# Skips files that already have liq_cue_file=true (use -f to force).
|
||||
local failed=0
|
||||
for f in "${files[@]}"; do
|
||||
if ! cue_file -w -k "$f" >>"$log" 2>&1; then
|
||||
echo "[prep-audio] cue_file failed on $f" >>"$log"
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
done
|
||||
[[ $failed -gt 0 ]] && echo "[prep-audio] cue_file errors: $failed" >>"$log"
|
||||
|
||||
echo "${#files[@]}"
|
||||
}
|
||||
Executable
+315
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
recover-azuracast-playlists.py — recover playlist assignment for files that
|
||||
lost their GROUPING tag (e.g. from a pre-tag-copy upgrade-mp3-to-flac.sh run).
|
||||
|
||||
For each AzuraCast file currently unassigned to any playlist:
|
||||
1. Resolve the source playlist by matching (artist, title) against the
|
||||
Spotify playlists named in $ALEMBIC_CONFIG_DIR/pipeline/*.conf.
|
||||
2. Write GROUPING=<playlist> onto the FLAC/MP3 file.
|
||||
3. `beet update` so beets sees the new tag.
|
||||
4. Assign the file to the matching AzuraCast playlist via API.
|
||||
|
||||
Default is dry-run. Pass --apply to actually write tags + assign.
|
||||
|
||||
Env / args:
|
||||
--azuracast-key KEY AzuraCast API key (required)
|
||||
--azuracast-base URL default https://admin.ephemeral.club
|
||||
--station N default 1
|
||||
--apply actually act (default is dry-run)
|
||||
--only PLAYLIST limit recovery to one playlist (testing)
|
||||
"""
|
||||
import argparse, base64, json, os, re, subprocess, sys, urllib.parse, urllib.request
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
|
||||
CONF_DIR = Path(f"{os.environ.get('ALEMBIC_CONFIG_DIR', '/config')}/pipeline")
|
||||
LIB_ROOT_HOST = f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library"
|
||||
# Still "/music" through the migration's Stage 0-3 transitional beets mount;
|
||||
# revisit at Stage 4 once beets' directory: becomes MUSIC_DATA_DIR/Library.
|
||||
LIB_ROOT_CONTAINER = "/music"
|
||||
|
||||
|
||||
def normalize(s: str) -> str:
|
||||
s = (s or "").lower()
|
||||
s = re.sub(r"\s*\([^)]*\)", "", s)
|
||||
s = re.sub(r"\s*\[[^\]]*\]", "", s)
|
||||
s = re.sub(r"\s*-?\s*(extended|original|radio|club|vip|vocal)\s*(mix|edit|version)\s*$", "", s)
|
||||
s = re.sub(r"\s*feat\.?\s.*", "", s)
|
||||
s = re.sub(r"\s*ft\.?\s.*", "", s)
|
||||
s = re.sub(r"[^a-z0-9]+", "", s)
|
||||
return s
|
||||
|
||||
|
||||
def read_conf(path: Path) -> dict:
|
||||
out = {}
|
||||
for line in path.read_text().splitlines():
|
||||
m = re.match(r"\s*([a-z0-9_-]+)\s*=\s*(.*?)\s*$", line)
|
||||
if m: out[m.group(1)] = m.group(2)
|
||||
return out
|
||||
|
||||
|
||||
def spotify_token(cid: str, csec: str) -> str:
|
||||
creds = base64.b64encode(f"{cid}:{csec}".encode()).decode()
|
||||
req = urllib.request.Request(
|
||||
"https://accounts.spotify.com/api/token",
|
||||
data=urllib.parse.urlencode({"grant_type": "client_credentials"}).encode(),
|
||||
headers={"Authorization": f"Basic {creds}",
|
||||
"Content-Type": "application/x-www-form-urlencoded"})
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
return json.loads(r.read())["access_token"]
|
||||
|
||||
|
||||
def fetch_spotify_playlist(url: str, token: str) -> list[tuple[str, str]]:
|
||||
"""Return list of (artists_normalized_joined, title_normalized)."""
|
||||
m = re.search(r"playlist[/:]([A-Za-z0-9]+)", url)
|
||||
if not m: return []
|
||||
pid = m.group(1)
|
||||
out = []
|
||||
next_url = f"https://api.spotify.com/v1/playlists/{pid}/tracks?limit=100"
|
||||
while next_url:
|
||||
req = urllib.request.Request(next_url, headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(req, timeout=20) as r:
|
||||
data = json.loads(r.read())
|
||||
for item in data["items"]:
|
||||
t = item.get("track")
|
||||
if not t or t.get("is_local"): continue
|
||||
artists = [a["name"] for a in t["artists"]]
|
||||
out.append((artists, t["name"]))
|
||||
next_url = data.get("next")
|
||||
return out
|
||||
|
||||
|
||||
def beets_dump() -> dict:
|
||||
"""Return path → {grouping, artist, albumartist, album, title}"""
|
||||
proc = subprocess.run(
|
||||
["beet", "ls", "-f",
|
||||
"$path\t$grouping\t$artist\t$albumartist\t$album\t$title"],
|
||||
capture_output=True, text=True, check=True)
|
||||
out = {}
|
||||
for line in proc.stdout.splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 6: continue
|
||||
out[parts[0]] = {
|
||||
"grouping": parts[1],
|
||||
"artist": parts[2],
|
||||
"albumartist": parts[3],
|
||||
"album": parts[4],
|
||||
"title": parts[5],
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def az_request(base: str, key: str, path: str, method="GET", body=None) -> dict:
|
||||
url = f"{base}{path}"
|
||||
headers = {"X-API-Key": key, "Accept": "application/json"}
|
||||
data = None
|
||||
if body is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
data = json.dumps(body).encode()
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
raw = r.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def az_all_files(base: str, key: str, station: int) -> list[dict]:
|
||||
out = []
|
||||
page = 1
|
||||
while True:
|
||||
d = az_request(base, key, f"/api/station/{station}/files?per_page=500&page={page}")
|
||||
out.extend(d["rows"])
|
||||
if page >= d["total_pages"]: break
|
||||
page += 1
|
||||
return out
|
||||
|
||||
|
||||
def az_playlists_by_name(base: str, key: str, station: int) -> dict:
|
||||
return {p["name"]: p["id"]
|
||||
for p in az_request(base, key, f"/api/station/{station}/playlists")}
|
||||
|
||||
|
||||
def write_grouping(host_path: str, grouping: str) -> bool:
|
||||
"""Write GROUPING tag in-place. Return True on success."""
|
||||
ext = host_path.rsplit(".", 1)[-1].lower()
|
||||
if ext == "flac":
|
||||
subprocess.run(["metaflac", "--remove-tag=GROUPING", host_path],
|
||||
capture_output=True, check=False)
|
||||
return subprocess.run(["metaflac", f"--set-tag=GROUPING={grouping}", host_path],
|
||||
capture_output=True).returncode == 0
|
||||
if ext == "mp3":
|
||||
return subprocess.run(["id3v2", "--TIT1", grouping, host_path],
|
||||
capture_output=True).returncode == 0
|
||||
return False
|
||||
|
||||
|
||||
def beet_update(container_paths: list[str], log) -> None:
|
||||
"""Re-read tags from disk into beets DB for the given container paths.
|
||||
Uses `path:<exact>` query (not `path::<regex>`) and pipes "y" since
|
||||
beet update prompts interactively for each change."""
|
||||
if not container_paths: return
|
||||
for cp in container_paths:
|
||||
subprocess.run(
|
||||
["beet", "update", "--nomove", f"path:{cp}"],
|
||||
input="y\n", capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--azuracast-key", required=True)
|
||||
ap.add_argument("--azuracast-base", default="https://admin.ephemeral.club")
|
||||
ap.add_argument("--station", type=int, default=1)
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--only", default=None, help="limit recovery to one playlist name")
|
||||
args = ap.parse_args()
|
||||
|
||||
print("[1/5] Loading beets metadata...", flush=True)
|
||||
beets = beets_dump()
|
||||
print(f" {len(beets)} tracks in beets")
|
||||
|
||||
print("[2/5] Fetching Spotify playlist contents...", flush=True)
|
||||
# Build (title_norm) → list of (playlist_name, artists_list)
|
||||
title_idx: dict[str, list[tuple[str, list[str]]]] = {}
|
||||
confs = sorted(CONF_DIR.glob("*.conf"))
|
||||
confs = [c for c in confs if not c.name.startswith("_")]
|
||||
if not confs:
|
||||
print(f"ERROR: no *.conf files in {CONF_DIR}/", file=sys.stderr); return 1
|
||||
first = read_conf(confs[0])
|
||||
token = spotify_token(first["spotify-id"], first["spotify-secret"])
|
||||
pl_count = 0
|
||||
for c in confs:
|
||||
cfg = read_conf(c)
|
||||
url = cfg.get("input"); pname = c.stem
|
||||
if not url or not url.startswith("https://open.spotify.com/"): continue
|
||||
try:
|
||||
tracks = fetch_spotify_playlist(url, token)
|
||||
except Exception as e:
|
||||
print(f" WARN: {pname}: {e}"); continue
|
||||
for artists, title in tracks:
|
||||
title_idx.setdefault(normalize(title), []).append((pname, artists))
|
||||
pl_count += 1
|
||||
print(f" {pname}: {len(tracks)} tracks")
|
||||
print(f" indexed {pl_count} playlists, {sum(len(v) for v in title_idx.values())} title→playlist entries")
|
||||
|
||||
print("[3/5] Fetching AzuraCast playlists + files...", flush=True)
|
||||
playlists_by_name = az_playlists_by_name(args.azuracast_base, args.azuracast_key, args.station)
|
||||
print(f" {len(playlists_by_name)} AzuraCast playlists")
|
||||
all_files = az_all_files(args.azuracast_base, args.azuracast_key, args.station)
|
||||
unassigned = [r for r in all_files if not r.get("playlists")]
|
||||
print(f" {len(all_files)} total files, {len(unassigned)} unassigned")
|
||||
|
||||
print("[4/5] Matching unassigned files to playlists...", flush=True)
|
||||
matches: list[dict] = [] # rows with .target_playlist
|
||||
unmatched: list[dict] = []
|
||||
multi_match: list[dict] = [] # ambiguous
|
||||
no_az_playlist: list[dict] = []
|
||||
|
||||
for r in unassigned:
|
||||
cp = f"{LIB_ROOT_CONTAINER}/{r['path']}"
|
||||
bmeta = beets.get(cp, {})
|
||||
# 1) trust beets GROUPING if present
|
||||
target = bmeta.get("grouping", "").strip() if bmeta else ""
|
||||
source = "beets_grouping" if target else None
|
||||
|
||||
if not target:
|
||||
# 2) resolve via Spotify (artist, title) match
|
||||
title_n = normalize(bmeta.get("title") or r.get("title") or "")
|
||||
file_artists = bmeta.get("artist") or r.get("artist") or ""
|
||||
candidates = title_idx.get(title_n, [])
|
||||
if not candidates:
|
||||
unmatched.append({"id": r["id"], "path": r["path"], "reason": "no spotify title hit"})
|
||||
continue
|
||||
# Score candidates by artist overlap
|
||||
file_artist_n = normalize(file_artists)
|
||||
scored = []
|
||||
for pname, artists in candidates:
|
||||
joined_n = normalize("; ".join(artists))
|
||||
score = SequenceMatcher(None, file_artist_n, joined_n).ratio()
|
||||
scored.append((score, pname))
|
||||
scored.sort(reverse=True)
|
||||
top_score, top_pname = scored[0]
|
||||
if top_score < 0.4:
|
||||
unmatched.append({"id": r["id"], "path": r["path"], "reason": f"low artist score {top_score:.2f}"})
|
||||
continue
|
||||
# If second-best is within 0.05, it's ambiguous between playlists
|
||||
distinct_playlists = {p for _, p in scored}
|
||||
if len(distinct_playlists) > 1 and len(scored) > 1 and scored[1][0] > top_score - 0.05 and scored[1][1] != top_pname:
|
||||
multi_match.append({"id": r["id"], "path": r["path"], "candidates": [p for _, p in scored[:3]]})
|
||||
continue
|
||||
target = top_pname
|
||||
source = "spotify_match"
|
||||
|
||||
if args.only and target != args.only:
|
||||
continue
|
||||
if target not in playlists_by_name:
|
||||
no_az_playlist.append({"id": r["id"], "path": r["path"], "target": target})
|
||||
continue
|
||||
|
||||
matches.append({
|
||||
"id": r["id"], "path": r["path"], "target": target, "source": source,
|
||||
"host_path": f"{LIB_ROOT_HOST}/{r['path']}",
|
||||
"container_path": cp,
|
||||
"playlist_id": playlists_by_name[target],
|
||||
})
|
||||
|
||||
# Report
|
||||
by_target: dict[str, int] = {}
|
||||
for m in matches:
|
||||
by_target[m["target"]] = by_target.get(m["target"], 0) + 1
|
||||
print()
|
||||
print(f"=== Match summary ===")
|
||||
print(f" matched: {len(matches)}")
|
||||
for pname, n in sorted(by_target.items(), key=lambda x: -x[1]):
|
||||
print(f" {pname}: {n}")
|
||||
print(f" unmatched (no spotify hit): {len(unmatched)}")
|
||||
print(f" ambiguous (multiple playlists tied): {len(multi_match)}")
|
||||
print(f" no matching AzuraCast playlist: {len(no_az_playlist)}")
|
||||
if unmatched[:5]:
|
||||
print(" unmatched sample:")
|
||||
for u in unmatched[:5]:
|
||||
print(f" {u['path']} ({u['reason']})")
|
||||
if multi_match[:5]:
|
||||
print(" ambiguous sample:")
|
||||
for u in multi_match[:5]:
|
||||
print(f" {u['path']} candidates={u['candidates']}")
|
||||
|
||||
if not args.apply:
|
||||
print()
|
||||
print("DRY RUN — pass --apply to (a) write GROUPING tags, (b) beet update, (c) assign in AzuraCast")
|
||||
return 0
|
||||
|
||||
print()
|
||||
print("[5/5] Applying...", flush=True)
|
||||
tag_ok = 0; tag_fail = 0
|
||||
az_ok = 0; az_fail = 0
|
||||
for m in matches:
|
||||
# Write tag
|
||||
if write_grouping(m["host_path"], m["target"]):
|
||||
tag_ok += 1
|
||||
else:
|
||||
tag_fail += 1
|
||||
print(f" TAG-FAIL {m['host_path']}")
|
||||
# Assign in AzuraCast: PUT /api/station/N/file/{id} with playlists field.
|
||||
# Send ONLY the playlists key — sending the full record back causes
|
||||
# AzuraCast to reject with HTTP 500 (validation fails on derived fields).
|
||||
try:
|
||||
az_request(args.azuracast_base, args.azuracast_key,
|
||||
f"/api/station/{args.station}/file/{m['id']}",
|
||||
method="PUT", body={"playlists": [{"id": m["playlist_id"]}]})
|
||||
az_ok += 1
|
||||
except Exception as e:
|
||||
az_fail += 1
|
||||
print(f" AZ-FAIL {m['path']}: {e}")
|
||||
|
||||
print(f" tags written: {tag_ok} (failed: {tag_fail})")
|
||||
print(f" azuracast assigned: {az_ok} (failed: {az_fail})")
|
||||
|
||||
# beet update so beets DB sees the new GROUPING
|
||||
print(" running beet update on changed paths...")
|
||||
beet_update([m["container_path"] for m in matches], None)
|
||||
print(" done")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+243
@@ -0,0 +1,243 @@
|
||||
#!/bin/bash
|
||||
# Walk a "stranded" download directory (files beets rejected as duplicates),
|
||||
# and for each file, compare against the existing library entry. If the new
|
||||
# file is higher quality (FLAC > MP3, then larger size), REPLACE the library
|
||||
# version. Otherwise delete the new file.
|
||||
#
|
||||
# Usage:
|
||||
# replace-with-better.sh # default scan: ${MUSIC_DATA_DIR:-/data/music}/downloads
|
||||
# replace-with-better.sh <scan_dir> # custom scan dir
|
||||
# replace-with-better.sh --apply # actually replace (default is dry-run)
|
||||
#
|
||||
# Run after sync-bandcamp.sh when the FLAC > MP3 dedup didn't pick up new
|
||||
# imports (beets quietly skipped them and left files in /downloads).
|
||||
|
||||
set -u
|
||||
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
|
||||
APPLY=0
|
||||
COPY_TAGS=0
|
||||
SCAN_DIR=${MUSIC_DATA_DIR:-/data/music}/downloads
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--apply) APPLY=1 ;;
|
||||
--copy-tags-from-existing) COPY_TAGS=1 ;;
|
||||
*) SCAN_DIR="$arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Copy canonical tags from an existing (already spotify-retagged) library track
|
||||
# onto a new FLAC, replacing whatever uploader-supplied tags it came with.
|
||||
# Uses beets as the source of truth — same fields beets/Navidrome care about.
|
||||
copy_tags_from_existing() {
|
||||
local existing_id="$1" # beets item id of the existing library track
|
||||
local new_file="$2" # host path of new FLAC
|
||||
[[ "${new_file,,}" == *.flac ]] || return 0
|
||||
# \x1f separator, NOT \t: tab is IFS whitespace, so consecutive tabs
|
||||
# (an empty field like $genres) collapse and shift every later field left —
|
||||
# GROUPING landing in GENRE etc. \x1f is non-whitespace, so empty fields
|
||||
# survive the read. Query by id: path queries are unreliable against the
|
||||
# mixed absolute/relative storage the beets 2.11 upgrade left in the DB.
|
||||
local raw
|
||||
raw=$(beet ls -f \
|
||||
$'$artist\x1f$albumartist\x1f$album\x1f$title\x1f$track\x1f$disc\x1f$year\x1f$genres\x1f$grouping' \
|
||||
"id:${existing_id}" 2>/dev/null | head -1)
|
||||
[[ -z "$raw" ]] && return 0
|
||||
local artist albumartist album title track disc year genre grouping
|
||||
IFS=$'\x1f' read -r artist albumartist album title track disc year genre grouping <<< "$raw"
|
||||
local tag
|
||||
for tag in ARTIST ALBUMARTIST ALBUM TITLE TRACKNUMBER DISCNUMBER DATE GENRE GROUPING; do
|
||||
metaflac --remove-tag="$tag" "$new_file" 2>/dev/null
|
||||
done
|
||||
[[ -n "$artist" ]] && metaflac --set-tag="ARTIST=$artist" "$new_file" 2>/dev/null
|
||||
[[ -n "$albumartist" ]] && metaflac --set-tag="ALBUMARTIST=$albumartist" "$new_file" 2>/dev/null
|
||||
[[ -n "$album" ]] && metaflac --set-tag="ALBUM=$album" "$new_file" 2>/dev/null
|
||||
[[ -n "$title" ]] && metaflac --set-tag="TITLE=$title" "$new_file" 2>/dev/null
|
||||
[[ -n "$track" && "$track" != "0" ]] && metaflac --set-tag="TRACKNUMBER=$track" "$new_file" 2>/dev/null
|
||||
[[ -n "$disc" && "$disc" != "0" ]] && metaflac --set-tag="DISCNUMBER=$disc" "$new_file" 2>/dev/null
|
||||
[[ -n "$year" && "$year" != "0" ]] && metaflac --set-tag="DATE=$year" "$new_file" 2>/dev/null
|
||||
[[ -n "$genre" ]] && metaflac --set-tag="GENRE=$genre" "$new_file" 2>/dev/null
|
||||
[[ -n "$grouping" ]] && metaflac --set-tag="GROUPING=$grouping" "$new_file" 2>/dev/null
|
||||
}
|
||||
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/replace-$(date +%Y%m%d-%H%M%S).log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
echo "[$(date -Iseconds)] === REPLACE (APPLY) scan=$SCAN_DIR ===" | tee -a "$LOG"
|
||||
# Radio prep (ReplayGain + autocue liq_* tags) BEFORE any import. Every
|
||||
# other import path preps first (run-playlist.sh, import-track.sh); this
|
||||
# one didn't, so replacement FLACs and leftover imports reached the library
|
||||
# unprepped for AzuraCast (found 2026-07-02: the July upgrade FLACs and a
|
||||
# 39-track leftover batch all lacked RG/autocue). Apply-mode only — dry
|
||||
# runs must not modify files.
|
||||
PREPPED=$(prep_audio "$SCAN_DIR" "$LOG" || echo 0)
|
||||
echo "[prep-audio] pre-tagged $PREPPED file(s) with ReplayGain + autocue" | tee -a "$LOG"
|
||||
else
|
||||
echo "[$(date -Iseconds)] === REPLACE (DRY RUN — pass --apply to act) scan=$SCAN_DIR ===" | tee -a "$LOG"
|
||||
fi
|
||||
|
||||
# A FLAC file always beats an MP3 file regardless of size. Among same-format
|
||||
# files, larger wins. Returns "new" or "existing".
|
||||
better() {
|
||||
local new="$1" existing="$2"
|
||||
local new_flac=0 ex_flac=0
|
||||
[[ "${new,,}" == *.flac ]] && new_flac=1
|
||||
[[ "${existing,,}" == *.flac ]] && ex_flac=1
|
||||
if [[ $new_flac -ne $ex_flac ]]; then
|
||||
if [[ $new_flac -eq 1 ]]; then echo "new"; else echo "existing"; fi
|
||||
return
|
||||
fi
|
||||
local ns es
|
||||
ns=$(stat -c '%s' "$new" 2>/dev/null || echo 0)
|
||||
es=$(stat -c '%s' "$existing" 2>/dev/null || echo 0)
|
||||
if [[ $ns -gt $es ]]; then echo "new"; else echo "existing"; fi
|
||||
}
|
||||
|
||||
# Read tag from FLAC or MP3 — return artist/album/title joined with newlines.
|
||||
read_tags() {
|
||||
local f="$1"
|
||||
case "${f,,}" in
|
||||
*.flac)
|
||||
local artist album title
|
||||
artist=$(metaflac --show-tag=ALBUMARTIST "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
[[ -z "$artist" ]] && artist=$(metaflac --show-tag=ARTIST "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
album=$(metaflac --show-tag=ALBUM "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
title=$(metaflac --show-tag=TITLE "$f" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
printf "%s\n%s\n%s\n" "$artist" "$album" "$title"
|
||||
;;
|
||||
*.mp3)
|
||||
local artist album title
|
||||
artist=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TPE2[^:]*: //p' | head -1)
|
||||
[[ -z "$artist" ]] && artist=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TPE1[^:]*: //p' | head -1)
|
||||
album=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TALB[^:]*: //p' | head -1)
|
||||
title=$(id3v2 -l "$f" 2>/dev/null | sed -n 's/^TIT2[^:]*: //p' | head -1)
|
||||
printf "%s\n%s\n%s\n" "$artist" "$album" "$title"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Beets queries (case-insensitive exact-substring on a field). Try progressively
|
||||
# looser matches until we find exactly one library track to compare against.
|
||||
# Echoes "<id>\x1f<path>" of the unique match; "" if no unique match found.
|
||||
beet_find_match() {
|
||||
local artist="$1" album="$2" title="$3"
|
||||
# Sanitize quotes
|
||||
artist="${artist//\"/}"; album="${album//\"/}"; title="${title//\"/}"
|
||||
|
||||
local fmt=$'$id\x1f$path'
|
||||
local hits
|
||||
# 1. Strictest: albumartist + album + title
|
||||
hits=$(beet ls -f "$fmt" \
|
||||
"albumartist:$artist" "album:$album" "title:$title" 2>/dev/null)
|
||||
[[ "$(echo "$hits" | wc -l)" == "1" && -n "$hits" ]] && { echo "$hits"; return; }
|
||||
|
||||
# 2. Drop album: maybe Bandcamp's "Artist | Album" doesn't match library's "Album"
|
||||
hits=$(beet ls -f "$fmt" \
|
||||
"albumartist:$artist" "title:$title" 2>/dev/null)
|
||||
[[ "$(echo "$hits" | wc -l)" == "1" && -n "$hits" ]] && { echo "$hits"; return; }
|
||||
|
||||
# 3. albumartist may be empty in old library entries — fall back to artist field
|
||||
hits=$(beet ls -f "$fmt" \
|
||||
"artist:$artist" "title:$title" 2>/dev/null)
|
||||
[[ "$(echo "$hits" | wc -l)" == "1" && -n "$hits" ]] && { echo "$hits"; return; }
|
||||
|
||||
# 4. Last resort: title alone — anchored exact match (case-insensitive),
|
||||
# NOT the substring match levels 1-3 use: a new track titled "Home"
|
||||
# must not "uniquely" match a library track titled "Coming Home Tonight"
|
||||
# and get it deleted. Only safe if exactly one library track carries
|
||||
# exactly this title.
|
||||
local escaped_title
|
||||
escaped_title=$(python3 -c \
|
||||
"import re,sys; print(re.escape(sys.argv[1]))" "$title" 2>/dev/null || echo "")
|
||||
if [[ -n "$escaped_title" ]]; then
|
||||
hits=$(beet ls -f "$fmt" \
|
||||
"title::(?i)^${escaped_title}\$" 2>/dev/null)
|
||||
[[ "$(echo "$hits" | wc -l)" == "1" && -n "$hits" ]] && { echo "$hits"; return; }
|
||||
fi
|
||||
|
||||
echo "" # no unique match
|
||||
}
|
||||
|
||||
REPLACED=0
|
||||
KEPT=0
|
||||
SKIPPED_NOMATCH=0
|
||||
|
||||
while IFS= read -r -d '' new_file; do
|
||||
# mapfile preserves empty lines (read with IFS=$'\n' would collapse them)
|
||||
mapfile -t _fields < <(read_tags "$new_file")
|
||||
artist="${_fields[0]:-}"
|
||||
album="${_fields[1]:-}"
|
||||
title="${_fields[2]:-}"
|
||||
if [[ -z "$artist" || -z "$title" ]]; then
|
||||
echo "[skip-untagged] $new_file" | tee -a "$LOG"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Find the matching library track via beets (progressive loosening)
|
||||
match=$(beet_find_match "$artist" "$album" "$title" | head -1)
|
||||
if [[ -z "$match" || "$match" != *$'\x1f'* ]]; then
|
||||
echo "[no-match] $artist - $album - $title (file: $new_file)" | tee -a "$LOG"
|
||||
SKIPPED_NOMATCH=$((SKIPPED_NOMATCH + 1))
|
||||
continue
|
||||
fi
|
||||
existing_id="${match%%$'\x1f'*}"
|
||||
existing_container="${match#*$'\x1f'}"
|
||||
|
||||
existing="${MUSIC_DATA_DIR:-/data/music}/Library${existing_container#/music}"
|
||||
if [[ ! -f "$existing" ]]; then
|
||||
echo "[stale-db] beets has $existing_container but file missing" | tee -a "$LOG"
|
||||
continue
|
||||
fi
|
||||
|
||||
winner=$(better "$new_file" "$existing")
|
||||
ns=$(numfmt --to=iec --suffix=B "$(stat -c '%s' "$new_file" 2>/dev/null || echo 0)")
|
||||
es=$(numfmt --to=iec --suffix=B "$(stat -c '%s' "$existing" 2>/dev/null || echo 0)")
|
||||
|
||||
if [[ "$winner" == "new" ]]; then
|
||||
echo "[REPLACE] $artist - $title ($es → $ns)" | tee -a "$LOG"
|
||||
echo " ex: $existing" | tee -a "$LOG"
|
||||
echo " new: $new_file" | tee -a "$LOG"
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
# If asked, copy canonical tags from the existing track to the new FLAC
|
||||
# BEFORE removing the existing entry. Used by upgrade-mp3-to-flac.sh so
|
||||
# the new FLAC inherits the MP3's spotify-retagged metadata (incl. the
|
||||
# GROUPING tag that drives playlist M3U regeneration).
|
||||
if [[ $COPY_TAGS -eq 1 ]]; then
|
||||
copy_tags_from_existing "$existing_id" "$new_file" >> "$LOG" 2>&1
|
||||
fi
|
||||
# Remove old library file + beets DB entry, then re-import the new one.
|
||||
# id query, not path: — see copy_tags_from_existing for why.
|
||||
beet remove -d -f "id:$existing_id" >> "$LOG" 2>&1
|
||||
beet import -q -s "$new_file" >> "$LOG" 2>&1
|
||||
fi
|
||||
REPLACED=$((REPLACED + 1))
|
||||
else
|
||||
echo "[KEEP] $artist - $title (existing $es ≥ new $ns)" | tee -a "$LOG"
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
rm -f "$new_file"
|
||||
fi
|
||||
KEPT=$((KEPT + 1))
|
||||
fi
|
||||
|
||||
done < <(find "$SCAN_DIR" -type f \( -iname "*.flac" -o -iname "*.mp3" \) -print0 2>/dev/null)
|
||||
|
||||
echo "" | tee -a "$LOG"
|
||||
echo "=== Summary: $REPLACED replaced, $KEPT existing-kept, $SKIPPED_NOMATCH no-match ===" | tee -a "$LOG"
|
||||
[[ $APPLY -eq 0 ]] && echo "DRY RUN — re-run with --apply to act" | tee -a "$LOG"
|
||||
|
||||
# After replaces, anything left in the scan dir is genuinely NEW (no library
|
||||
# match — not a duplicate of anything we already have). Import it as a new
|
||||
# track. Beets path templates will place it under /music/<albumartist>/...
|
||||
# alongside everything else.
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
remaining=$(find "$SCAN_DIR" -type f \( -iname "*.flac" -o -iname "*.mp3" \) 2>/dev/null | wc -l)
|
||||
if [[ "$remaining" -gt 0 ]]; then
|
||||
echo "[import-leftover] $remaining new file(s) under $SCAN_DIR — importing as fresh tracks" | tee -a "$LOG"
|
||||
beet import -q -s "$SCAN_DIR" >> "$LOG" 2>&1
|
||||
fi
|
||||
find "$SCAN_DIR" -mindepth 1 -type d -empty -delete 2>>"$LOG"
|
||||
echo "[cleanup] removed empty subfolders under $SCAN_DIR" | tee -a "$LOG"
|
||||
fi
|
||||
Executable
+208
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
scrub-watermark-text.py — strip Soulseek-uploader watermark URLs from
|
||||
metadata text frames across the library. Some uploaders pollute every
|
||||
available field (COMM, USLT, TXXX, WXXX, even TCON/genre) with their
|
||||
domain, so a track might have 8+ watermark frames each pointing at
|
||||
electronicfresh.com / djsoundtop.com / etc.
|
||||
|
||||
Targets — for any frame that contains a URL or known watermark domain:
|
||||
MP3 (ID3v2):
|
||||
COMM: comments
|
||||
USLT: unsynchronized lyrics
|
||||
TXXX: user-defined text (kept if value is non-URL)
|
||||
WXXX, WCOM, WPAY, WPUB, WORS, WCOP: URL link frames (always stripped)
|
||||
TCOP: copyright
|
||||
TPUB: publisher
|
||||
TENC: encoded by
|
||||
TCON: genre (only stripped if it's a URL — real genres are short text)
|
||||
FLAC (Vorbis):
|
||||
COMMENT, DESCRIPTION, COPYRIGHT, ENCODED-BY, CONTACT, WEBPAGE,
|
||||
LICENSE — wiped if value contains URL pattern
|
||||
GENRE — wiped only if it looks like a URL
|
||||
|
||||
Frames that hold real metadata we always keep:
|
||||
TIT2/TIT1/TPE1/TPE2/TALB/TPOS/TRCK/TDRC/TYER/TXXX:GROUPING (ours)
|
||||
ARTIST/ALBUMARTIST/ALBUM/TITLE/TRACKNUMBER/DISCNUMBER/DATE/GROUPING
|
||||
|
||||
Usage:
|
||||
scrub-watermark-text.py # dry run
|
||||
scrub-watermark-text.py --apply # actually strip
|
||||
"""
|
||||
import sys, re, argparse
|
||||
from pathlib import Path
|
||||
from mutagen import File as MFile
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError
|
||||
from mutagen.flac import FLAC
|
||||
|
||||
LIBRARY = f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library"
|
||||
|
||||
# Anything containing one of these is suspect. Add to taste.
|
||||
WATERMARK_DOMAINS = [
|
||||
"electronicfresh", "djsoundtop", "iptorrents", "torrentday",
|
||||
"ftpdjemilio", "hypeddit", "hypeddit.com", "djseamusic", "djpool",
|
||||
"myzuka", "promodj",
|
||||
]
|
||||
|
||||
# Generic URL pattern — http(s)://, www., or a bare domain.tld
|
||||
URL_RE = re.compile(
|
||||
r"(?ix)"
|
||||
r"(?:https?://|www\.|[a-z0-9-]+\.(?:com|net|org|io|to|cc|me|tv|fm|gg|co|sh|ru))"
|
||||
r"[a-z0-9./?#=&_+:%-]*"
|
||||
)
|
||||
|
||||
# Frames that should be FULLY removed if they contain ANY URL.
|
||||
# NOTE: WCOM ("commercial information") and WPAY ("payment") are deliberately
|
||||
# NOT stripped — that's where the buy-link pipeline stores the legit Bandcamp
|
||||
# purchase URL (see sync-bandcamp.py BUY_URL_TAG / README "Buy links"). On
|
||||
# FLAC the equivalent COMMERCIAL_INFORMATION Vorbis comment is already safe
|
||||
# (it isn't in FLAC_TEXT_FIELDS_TO_CHECK). Watermarkers overwhelmingly abuse
|
||||
# COMM/TXXX/WXXX/USLT, not WCOM/WPAY, so excluding these costs us nothing.
|
||||
MP3_URL_LINK_FRAMES = ("WXXX", "WPUB", "WORS", "WCOP", "WOAR", "WOAS", "WOAF")
|
||||
# Frames where we strip the WHOLE frame if its text content contains a URL.
|
||||
MP3_TEXT_FRAMES_TO_CHECK = ("COMM", "USLT", "TCOP", "TPUB", "TENC", "TXXX",
|
||||
"WXXX", "TOPE", "TOAL")
|
||||
# TCON is special — only strip if it's a URL (real genres are normal text)
|
||||
|
||||
|
||||
def text_has_url(s):
|
||||
if not s: return False
|
||||
s = str(s)
|
||||
if URL_RE.search(s):
|
||||
return True
|
||||
sl = s.lower()
|
||||
return any(d in sl for d in WATERMARK_DOMAINS)
|
||||
|
||||
|
||||
def scrub_mp3(path, apply):
|
||||
try:
|
||||
tags = ID3(path)
|
||||
except (ID3NoHeaderError, Exception):
|
||||
return 0, []
|
||||
|
||||
to_delete = []
|
||||
actions = []
|
||||
|
||||
for k in list(tags.keys()):
|
||||
prefix = k[:4]
|
||||
# URL link frames — strip if they have anything URL-y
|
||||
if prefix in MP3_URL_LINK_FRAMES:
|
||||
v = getattr(tags[k], "url", None) or str(tags[k])
|
||||
if text_has_url(v):
|
||||
to_delete.append(k)
|
||||
actions.append(f"remove {k}: {str(v)[:80]}")
|
||||
continue
|
||||
# Text frames — check the rendered text
|
||||
if prefix in MP3_TEXT_FRAMES_TO_CHECK:
|
||||
v = str(tags[k])
|
||||
if text_has_url(v):
|
||||
to_delete.append(k)
|
||||
actions.append(f"remove {k}: {v[:80]}")
|
||||
continue
|
||||
# TCON: genre. Strip if it's clearly a URL.
|
||||
if prefix == "TCON":
|
||||
v = str(tags[k])
|
||||
if text_has_url(v):
|
||||
to_delete.append(k)
|
||||
actions.append(f"remove TCON (was URL'd genre): {v[:80]}")
|
||||
continue
|
||||
|
||||
if apply and to_delete:
|
||||
for k in to_delete:
|
||||
del tags[k]
|
||||
try:
|
||||
tags.save(path)
|
||||
except Exception as e:
|
||||
actions.append(f" SAVE FAILED: {e}")
|
||||
return len(to_delete), actions
|
||||
|
||||
|
||||
# Vorbis comment fields (case-insensitive in spec, we handle that)
|
||||
FLAC_TEXT_FIELDS_TO_CHECK = (
|
||||
"comment", "description", "copyright", "encoded-by",
|
||||
"contact", "webpage", "website", "license", "url",
|
||||
)
|
||||
|
||||
|
||||
def scrub_flac(path, apply):
|
||||
try:
|
||||
f = FLAC(path)
|
||||
except Exception:
|
||||
return 0, []
|
||||
|
||||
to_delete = [] # list of (field_name, value)
|
||||
actions = []
|
||||
|
||||
for key in list(f.tags.keys() if f.tags else []):
|
||||
kl = key.lower()
|
||||
vals = f.tags[key]
|
||||
if kl in FLAC_TEXT_FIELDS_TO_CHECK:
|
||||
for v in vals:
|
||||
if text_has_url(v):
|
||||
to_delete.append((key, v))
|
||||
actions.append(f"remove {key}: {v[:80]}")
|
||||
elif kl == "genre":
|
||||
for v in vals:
|
||||
if text_has_url(v):
|
||||
to_delete.append((key, v))
|
||||
actions.append(f"remove genre (was URL'd): {v[:80]}")
|
||||
|
||||
if apply and to_delete:
|
||||
# Group by field, rebuild the field's value list without URL-y entries.
|
||||
fields_to_rewrite = {}
|
||||
for key, val in to_delete:
|
||||
fields_to_rewrite.setdefault(key, []).append(val)
|
||||
for key, bad_vals in fields_to_rewrite.items():
|
||||
current = list(f.tags[key])
|
||||
keep = [v for v in current if v not in bad_vals]
|
||||
if keep:
|
||||
f.tags[key] = keep
|
||||
else:
|
||||
del f.tags[key]
|
||||
try:
|
||||
f.save()
|
||||
except Exception as e:
|
||||
actions.append(f" SAVE FAILED: {e}")
|
||||
return len(to_delete), actions
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
print(f"[scrub-text] mode={'APPLY' if args.apply else 'DRY RUN'}")
|
||||
print(f"[scrub-text] watermark domains: {WATERMARK_DOMAINS}\n")
|
||||
|
||||
files = list(Path(LIBRARY).rglob("*.mp3")) + list(Path(LIBRARY).rglob("*.flac"))
|
||||
print(f"[scrub-text] scanning {len(files)} files...\n")
|
||||
|
||||
total_frames = 0
|
||||
files_touched = 0
|
||||
for i, p in enumerate(files, 1):
|
||||
if i % 200 == 0:
|
||||
print(f" ...{i}/{len(files)}", flush=True)
|
||||
sfx = p.suffix.lower()
|
||||
if sfx == ".mp3":
|
||||
n, actions = scrub_mp3(str(p), args.apply)
|
||||
elif sfx == ".flac":
|
||||
n, actions = scrub_flac(str(p), args.apply)
|
||||
else:
|
||||
continue
|
||||
if n > 0:
|
||||
files_touched += 1
|
||||
total_frames += n
|
||||
print(f"\n {p.name} ({n} frame(s))")
|
||||
for a in actions[:5]:
|
||||
print(f" {a}")
|
||||
if len(actions) > 5:
|
||||
print(f" ...and {len(actions) - 5} more")
|
||||
|
||||
verb = "stripped" if args.apply else "would-strip"
|
||||
print(f"\n[scrub-text] {verb} {total_frames} frame(s) across {files_touched} file(s)")
|
||||
if not args.apply:
|
||||
print("[scrub-text] re-run with --apply to write changes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+350
@@ -0,0 +1,350 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
spotify-genre.py — write GENRE tags onto every audio file in the library
|
||||
using Spotify's per-artist genre data, filtered against a whitelist.
|
||||
|
||||
Spotify only has genre at the ARTIST level, but their taxonomy is curated
|
||||
and clean (e.g. "tropical house", "tech house", "synthwave"). We match
|
||||
each artist's reported genres against our whitelist to pick up to 3
|
||||
canonical display strings, then write to every track by that artist.
|
||||
|
||||
Whitelist matching is fuzzy-normalized: lowercased, dashes/spaces stripped.
|
||||
So Spotify's "tropical house" matches our whitelist's "Tropical House"
|
||||
(or rolls into "House" if Tropical House isn't in the list).
|
||||
|
||||
Per-artist caching: each unique artist is looked up once, even if they
|
||||
have 100 tracks in the library.
|
||||
|
||||
Falls back gracefully:
|
||||
- Artist not on Spotify -> no change
|
||||
- Found but no whitelist match -> no change
|
||||
|
||||
Usage:
|
||||
spotify-genre.py # dry run, show plan
|
||||
spotify-genre.py --apply # write tags
|
||||
spotify-genre.py --apply --force # overwrite existing GENRE tags
|
||||
spotify-genre.py --apply --query '...' # subset by beets query
|
||||
"""
|
||||
import sys, os, re, json, argparse, base64, urllib.parse, urllib.request, subprocess
|
||||
from pathlib import Path
|
||||
from mutagen import File as MFile
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError, TCON
|
||||
from mutagen.flac import FLAC
|
||||
from mutagen.mp4 import MP4
|
||||
from mutagen.oggopus import OggOpus
|
||||
from mutagen.oggvorbis import OggVorbis
|
||||
|
||||
# OGG/OPUS use Vorbis-comment-style tags (lowercase keys, multi-value), same
|
||||
# shape as FLAC. Treat them as a family so the read/write paths are shared.
|
||||
VORBIS_LIKE = (FLAC, OggOpus, OggVorbis)
|
||||
|
||||
ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
|
||||
|
||||
|
||||
def _load_spotify_creds():
|
||||
"""Spotify creds, rendered by alembic's credential_service to _spotify.env."""
|
||||
path = f"{ALEMBIC_CONFIG_DIR}/pipeline/_spotify.env"
|
||||
env = {}
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env[k] = v.strip().strip("'").strip('"')
|
||||
return env["SPOTIFY_CLIENT_ID"], env["SPOTIFY_CLIENT_SECRET"]
|
||||
|
||||
SPOTIFY_CID, SPOTIFY_CSEC = _load_spotify_creds()
|
||||
WHITELIST_FILE = f"{ALEMBIC_CONFIG_DIR}/pipeline/genres-whitelist.txt"
|
||||
MAX_GENRES = 3
|
||||
SEPARATOR = "; "
|
||||
|
||||
|
||||
def load_whitelist():
|
||||
"""Load whitelist and build a normalized→display mapping for fuzzy matching."""
|
||||
out = {}
|
||||
with open(WHITELIST_FILE) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
normalized = re.sub(r"[\s\-_&/.]+", "", line).lower()
|
||||
out[normalized] = line
|
||||
return out
|
||||
|
||||
|
||||
_token = None
|
||||
def _spotify_token():
|
||||
global _token
|
||||
if _token: return _token
|
||||
creds = base64.b64encode(f"{SPOTIFY_CID}:{SPOTIFY_CSEC}".encode()).decode()
|
||||
req = urllib.request.Request(
|
||||
"https://accounts.spotify.com/api/token",
|
||||
data=urllib.parse.urlencode({"grant_type":"client_credentials"}).encode(),
|
||||
headers={"Authorization":f"Basic {creds}",
|
||||
"Content-Type":"application/x-www-form-urlencoded"})
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
_token = json.loads(r.read())["access_token"]
|
||||
return _token
|
||||
|
||||
|
||||
# Cache: artist_name → list[str] of Spotify genres (or None if not found)
|
||||
_artist_genre_cache = {}
|
||||
|
||||
|
||||
def spotify_artist_genres(artist_name):
|
||||
"""Return Spotify's genre list for the artist, or None if not found.
|
||||
Cached per-artist so 1500 tracks for ~300 artists = ~300 API calls."""
|
||||
if artist_name in _artist_genre_cache:
|
||||
return _artist_genre_cache[artist_name]
|
||||
|
||||
# Search artist by name (limit=1, take best match)
|
||||
qs = urllib.parse.urlencode({
|
||||
"q": f'artist:"{artist_name}"',
|
||||
"type": "artist",
|
||||
"limit": 1,
|
||||
})
|
||||
url = f"https://api.spotify.com/v1/search?{qs}"
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {_spotify_token()}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
data = json.loads(r.read())
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, TimeoutError):
|
||||
_artist_genre_cache[artist_name] = None
|
||||
return None
|
||||
|
||||
items = data.get("artists", {}).get("items", [])
|
||||
if not items:
|
||||
_artist_genre_cache[artist_name] = None
|
||||
return None
|
||||
|
||||
# Validate: Spotify's fuzzy search will silently match short/ambiguous
|
||||
# names to the wrong artist (e.g. "G Jones" → "George Jones" the country
|
||||
# singer, "T78" → some random T78). Reject the result unless the returned
|
||||
# name matches what we asked for after normalizing whitespace/punct.
|
||||
def _norm(s):
|
||||
return re.sub(r"[\s\-_&/.()\[\]'\"]+", "", s or "").lower()
|
||||
requested = _norm(artist_name)
|
||||
returned = _norm(items[0].get("name", ""))
|
||||
if requested and returned and requested != returned:
|
||||
_artist_genre_cache[artist_name] = None
|
||||
return None
|
||||
|
||||
genres = items[0].get("genres") or []
|
||||
_artist_genre_cache[artist_name] = genres
|
||||
return genres
|
||||
|
||||
|
||||
# Map Spotify-style spellings to the normalized form of their whitelist entry.
|
||||
# Spotify uses "drum and bass" / "rock and roll" / "r&b" — our whitelist uses
|
||||
# "Drum & Bass" / "Rock & Roll" / "R&B". The "and"/"&" difference prevents the
|
||||
# roll-up match below, so we explicitly resolve common variants first.
|
||||
ALIASES = {
|
||||
"drumandbass": "drumbass",
|
||||
"rockandroll": "rockroll",
|
||||
"randb": "rb",
|
||||
"rnb": "rb",
|
||||
"hiphop": "hiphop",
|
||||
"ukgarage": "ukgarage",
|
||||
}
|
||||
|
||||
|
||||
def match_whitelist(genres, whitelist):
|
||||
"""Pick up to MAX_GENRES whitelist entries from Spotify's genre list,
|
||||
in Spotify's reported order (most-relevant first). Returns list of
|
||||
display strings.
|
||||
|
||||
For specific genres like 'tropical house' that aren't in the whitelist,
|
||||
we also check if any whitelist entry's normalized form is contained
|
||||
within the genre's normalized form — so 'tropical house' rolls up to
|
||||
'House' if Tropical House isn't a separate whitelist entry."""
|
||||
out = []
|
||||
seen = set()
|
||||
for g in genres or []:
|
||||
norm = re.sub(r"[\s\-_&/.]+", "", g).lower()
|
||||
# Resolve known aliases (e.g. "drum and bass" → "drumbass") before lookup
|
||||
norm = ALIASES.get(norm, norm)
|
||||
# Direct hit
|
||||
display = whitelist.get(norm)
|
||||
if display:
|
||||
if display not in seen:
|
||||
out.append(display); seen.add(display)
|
||||
continue
|
||||
# Roll-up: e.g. "tropicalhouse" contains "house"
|
||||
for wl_norm, wl_display in whitelist.items():
|
||||
if wl_norm and wl_norm in norm and len(wl_norm) >= 4:
|
||||
if wl_display not in seen:
|
||||
out.append(wl_display); seen.add(wl_display)
|
||||
break
|
||||
if len(out) >= MAX_GENRES:
|
||||
break
|
||||
return out[:MAX_GENRES]
|
||||
|
||||
|
||||
def is_genre_locked(filepath):
|
||||
"""Return True if the file has GENRE_LOCK=1, meaning manual genre should not be overwritten."""
|
||||
try:
|
||||
m = MFile(filepath)
|
||||
except Exception:
|
||||
return False
|
||||
if not m or not m.tags:
|
||||
return False
|
||||
try:
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
val = (m.tags.get("genre_lock") or m.tags.get("GENRE_LOCK") or [None])[0]
|
||||
return val == "1"
|
||||
else:
|
||||
# MP3: check TXXX:GENRE_LOCK
|
||||
t = getattr(m, "tags", None)
|
||||
if t is not None:
|
||||
for frame in t.getall("TXXX"):
|
||||
if frame.desc.upper() == "GENRE_LOCK" and frame.text and frame.text[0] == "1":
|
||||
return True
|
||||
except (AttributeError, IndexError, TypeError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def get_track_meta(filepath):
|
||||
"""Read (artist, title, current_genre) from the file. Returns (None,None,None) on failure."""
|
||||
try:
|
||||
m = MFile(filepath)
|
||||
except Exception:
|
||||
return None, None, None
|
||||
if not m or not m.tags:
|
||||
return None, None, None
|
||||
|
||||
artist = title = genre = None
|
||||
try:
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
# FLAC/OGG/OPUS share Vorbis-comment-style tags
|
||||
artist = (m.tags.get("artist") or [None])[0]
|
||||
title = (m.tags.get("title") or [None])[0]
|
||||
genre = (m.tags.get("genre") or [None])[0]
|
||||
elif isinstance(m, MP4):
|
||||
artist = (m.tags.get("©ART") or [None])[0]
|
||||
title = (m.tags.get("©nam") or [None])[0]
|
||||
genre = (m.tags.get("©gen") or [None])[0]
|
||||
else:
|
||||
# MP3 ID3v2 — read via .text[0] of the frame to avoid str() returning frame metadata
|
||||
t = getattr(m, "tags", None)
|
||||
if t is not None:
|
||||
if "TPE1" in t and t["TPE1"].text: artist = str(t["TPE1"].text[0])
|
||||
if "TIT2" in t and t["TIT2"].text: title = str(t["TIT2"].text[0])
|
||||
if "TCON" in t and t["TCON"].text: genre = str(t["TCON"].text[0])
|
||||
except (AttributeError, IndexError, TypeError):
|
||||
pass
|
||||
return artist, title, genre
|
||||
|
||||
|
||||
def write_genre(filepath, value):
|
||||
"""Write GENRE tag on the file."""
|
||||
try:
|
||||
m = MFile(filepath)
|
||||
except Exception:
|
||||
return False
|
||||
if isinstance(m, VORBIS_LIKE):
|
||||
m.tags["GENRE"] = value # mutagen Vorbis-style tags are case-insensitive
|
||||
m.save()
|
||||
elif isinstance(m, MP4):
|
||||
m.tags["©gen"] = value
|
||||
m.save()
|
||||
else:
|
||||
try:
|
||||
tags = ID3(filepath)
|
||||
except ID3NoHeaderError:
|
||||
tags = ID3()
|
||||
tags.delall("TCON")
|
||||
tags.add(TCON(encoding=3, text=value))
|
||||
tags.save(filepath)
|
||||
return True
|
||||
|
||||
|
||||
def list_tracks(query=None):
|
||||
"""Use beets (in-process, same container) to list tracks; paths are
|
||||
already valid in this container's filesystem — no translation needed."""
|
||||
args = ["beet", "ls", "-f", "$path"]
|
||||
if query:
|
||||
args.append(query)
|
||||
r = subprocess.run(args, capture_output=True, text=True, timeout=120)
|
||||
return [line for line in r.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true", help="Actually write tags")
|
||||
ap.add_argument("--force", action="store_true",
|
||||
help="Overwrite existing GENRE tags (default: only fill empty)")
|
||||
ap.add_argument("--query", help="Beets query to limit which tracks are processed")
|
||||
ap.add_argument("--limit", type=int, help="Stop after N tracks (debug)")
|
||||
args = ap.parse_args()
|
||||
|
||||
whitelist = load_whitelist()
|
||||
print(f"[spotify-genre] whitelist has {len(whitelist)} entries")
|
||||
|
||||
paths = list_tracks(args.query)
|
||||
if args.limit:
|
||||
paths = paths[:args.limit]
|
||||
print(f"[spotify-genre] {len(paths)} tracks to scan")
|
||||
|
||||
written = unchanged = no_match = no_meta = 0
|
||||
for i, p in enumerate(paths, 1):
|
||||
if i % 100 == 0:
|
||||
print(f" ...{i}/{len(paths)}", flush=True)
|
||||
if not os.path.exists(p):
|
||||
continue
|
||||
|
||||
artist, title, current_genre = get_track_meta(p)
|
||||
if not artist or not title:
|
||||
no_meta += 1
|
||||
continue
|
||||
|
||||
# Skip if manually genre-locked (set via fix-genre.sh or direct GENRE_LOCK tag)
|
||||
if is_genre_locked(p):
|
||||
unchanged += 1
|
||||
continue
|
||||
|
||||
# Skip if already has whitelist-canonical genre and we're not forcing
|
||||
if current_genre and not args.force:
|
||||
cur_parts = [g.strip() for g in re.split(r"[;,]", current_genre)]
|
||||
cur_normed = {re.sub(r"[\s\-_&/.]+", "", g).lower() for g in cur_parts}
|
||||
if cur_normed & set(whitelist.keys()):
|
||||
unchanged += 1
|
||||
continue
|
||||
|
||||
# Multi-artist "X; Y" → use first as primary
|
||||
primary_artist = re.split(r"[;,/&]", artist)[0].strip()
|
||||
genres = spotify_artist_genres(primary_artist)
|
||||
if genres is None or not genres:
|
||||
no_match += 1
|
||||
continue
|
||||
|
||||
matched = match_whitelist(genres, whitelist)
|
||||
if not matched:
|
||||
no_match += 1
|
||||
continue
|
||||
|
||||
new_value = SEPARATOR.join(matched)
|
||||
if current_genre == new_value:
|
||||
unchanged += 1
|
||||
continue
|
||||
|
||||
if args.apply:
|
||||
ok = write_genre(p, new_value)
|
||||
if ok:
|
||||
written += 1
|
||||
# Quiet — only print every 50th
|
||||
if written % 50 == 0:
|
||||
print(f" [{written}] {primary_artist} - {title}: {new_value}", flush=True)
|
||||
else:
|
||||
print(f" WOULD-SET {primary_artist} - {title} → {new_value}")
|
||||
written += 1 # count as 'planned'
|
||||
|
||||
print(f"\n[spotify-genre] {('written' if args.apply else 'plan')}: {written}, "
|
||||
f"unchanged: {unchanged}, no-match: {no_match}, no-meta: {no_meta}")
|
||||
if args.apply:
|
||||
print("[spotify-genre] now run 'beet update' to sync the DB")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+282
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
spotify-retag.py — rewrite tags on downloaded files using Spotify as source of truth.
|
||||
|
||||
Reads a Spotify playlist's tracks via Client Credentials, matches each audio file
|
||||
in <scan_dir> to a Spotify track, and overwrites:
|
||||
|
||||
ARTIST — semicolon-joined list of all credited artists
|
||||
ALBUMARTIST — primary (first) artist only — kills ghost combined artists
|
||||
ALBUM
|
||||
TITLE
|
||||
TRACKNUMBER
|
||||
DISCNUMBER
|
||||
DATE — release year
|
||||
|
||||
Tags we leave alone:
|
||||
GROUPING — set by run-playlist.sh for playlist-membership tracking
|
||||
Anything else not in the list above
|
||||
|
||||
Usage:
|
||||
spotify-retag.py <playlist_url> <scan_dir> # walk a directory tree
|
||||
spotify-retag.py <playlist_url> - # read newline-separated paths from stdin
|
||||
|
||||
Credentials are read from env vars SPOTIFY_CLIENT_ID and SPOTIFY_CLIENT_SECRET.
|
||||
|
||||
Matching strategy (per file):
|
||||
1. Parse "Artist - Title.ext" from filename. If multiple " - " separators,
|
||||
try every possible split position.
|
||||
2. Normalize title (lowercase, strip parens/brackets/feat./punct).
|
||||
3. Look up by normalized title in the playlist index. If exactly one hit,
|
||||
use it. If multiple, score each by artist-string overlap and pick the
|
||||
best. If nothing close, leave the file alone.
|
||||
"""
|
||||
import sys, os, re, json, base64, urllib.request, urllib.parse, subprocess
|
||||
from pathlib import Path
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
API = "https://api.spotify.com/v1"
|
||||
|
||||
|
||||
def get_token(cid: str, csec: str) -> str:
|
||||
creds = base64.b64encode(f"{cid}:{csec}".encode()).decode()
|
||||
req = urllib.request.Request(
|
||||
"https://accounts.spotify.com/api/token",
|
||||
data=urllib.parse.urlencode({"grant_type": "client_credentials"}).encode(),
|
||||
headers={
|
||||
"Authorization": f"Basic {creds}",
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
return json.loads(r.read())["access_token"]
|
||||
|
||||
|
||||
def fetch_playlist(url: str, token: str) -> list[dict]:
|
||||
m = re.search(r"playlist[/:]([A-Za-z0-9]+)", url)
|
||||
if not m:
|
||||
sys.exit(f"Could not parse playlist ID from {url!r}")
|
||||
pid = m.group(1)
|
||||
tracks = []
|
||||
next_url = f"{API}/playlists/{pid}/tracks?limit=100"
|
||||
while next_url:
|
||||
req = urllib.request.Request(
|
||||
next_url, headers={"Authorization": f"Bearer {token}"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
data = json.loads(r.read())
|
||||
for item in data["items"]:
|
||||
t = item.get("track")
|
||||
if not t or t.get("is_local"):
|
||||
continue
|
||||
tracks.append(
|
||||
{
|
||||
"id": t["id"],
|
||||
"title": t["name"],
|
||||
"artists": [a["name"] for a in t["artists"]],
|
||||
"album": t["album"]["name"],
|
||||
"track_number": t.get("track_number") or 0,
|
||||
"disc_number": t.get("disc_number") or 1,
|
||||
"year": (t["album"].get("release_date") or "").split("-")[0],
|
||||
}
|
||||
)
|
||||
next_url = data.get("next")
|
||||
return tracks
|
||||
|
||||
|
||||
def normalize(s: str) -> str:
|
||||
s = s.lower()
|
||||
s = re.sub(r"\s*\([^)]*\)", "", s) # strip ( ... )
|
||||
s = re.sub(r"\s*\[[^\]]*\]", "", s) # strip [ ... ]
|
||||
s = re.sub(r"\s*-?\s*(extended|original|radio|club|vip|vocal)\s*(mix|edit|version)\s*$", "", s)
|
||||
s = re.sub(r"\s*feat\.?\s.*", "", s) # strip "feat. X" trailing
|
||||
s = re.sub(r"\s*ft\.?\s.*", "", s)
|
||||
s = re.sub(r"[^a-z0-9]+", "", s)
|
||||
return s
|
||||
|
||||
|
||||
def index_by_title(tracks: list[dict]) -> dict[str, list[dict]]:
|
||||
idx: dict[str, list[dict]] = {}
|
||||
for t in tracks:
|
||||
key = normalize(t["title"])
|
||||
# Fully non-Latin titles (Japanese/Korean/Cyrillic) normalize to "" —
|
||||
# indexing them would dump every such track into one shared bucket and
|
||||
# cross-match unrelated songs (their artists also normalize to "", so
|
||||
# SequenceMatcher scores them 1.0). Leave them unmatched instead; the
|
||||
# file keeps its sldl-supplied tags rather than gaining wrong ones.
|
||||
if not key:
|
||||
continue
|
||||
idx.setdefault(key, []).append(t)
|
||||
return idx
|
||||
|
||||
|
||||
def candidate_splits(stem: str) -> list[tuple[str, str]]:
|
||||
"""For 'A - B - C', return both forward and reversed splits, e.g.
|
||||
[('A','B - C'), ('A - B','C'), # forward: artist - title
|
||||
('B - C','A'), ('C','A - B')] # reversed: title - artist
|
||||
Some Soulseek uploaders write 'Title - Artist' instead of 'Artist - Title'.
|
||||
"""
|
||||
parts = stem.split(" - ")
|
||||
if len(parts) < 2:
|
||||
return []
|
||||
forward = [
|
||||
(" - ".join(parts[:i]), " - ".join(parts[i:])) for i in range(1, len(parts))
|
||||
]
|
||||
reversed_ = [(b, a) for a, b in forward]
|
||||
return forward + reversed_
|
||||
|
||||
|
||||
def best_match(file_artist_raw: str, candidates: list[dict]) -> dict | None:
|
||||
"""Pick the candidate whose artist-string overlaps best with the filename's."""
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
fnorm = normalize(file_artist_raw)
|
||||
best, best_score = None, 0.0
|
||||
for c in candidates:
|
||||
# Compare against the joined artist string
|
||||
joined = "; ".join(c["artists"])
|
||||
score = SequenceMatcher(None, fnorm, normalize(joined)).ratio()
|
||||
if score > best_score:
|
||||
best, best_score = c, score
|
||||
return best if best_score >= 0.4 else None
|
||||
|
||||
|
||||
def match_file(path: Path, idx: dict[str, list[dict]]) -> dict | None:
|
||||
stem = path.stem
|
||||
for artist_guess, title_guess in candidate_splits(stem):
|
||||
hits = idx.get(normalize(title_guess))
|
||||
if hits:
|
||||
picked = best_match(artist_guess, hits)
|
||||
if picked:
|
||||
return picked
|
||||
# Last resort: try the whole stem as title (e.g. "Title.flac" with no " - ")
|
||||
hits = idx.get(normalize(stem))
|
||||
if hits and len(hits) == 1:
|
||||
return hits[0]
|
||||
return None
|
||||
|
||||
|
||||
def write_flac_tags(path: str, t: dict, log) -> None:
|
||||
artists_joined = "; ".join(t["artists"])
|
||||
primary = t["artists"][0]
|
||||
pairs = {
|
||||
"ARTIST": artists_joined,
|
||||
"ALBUMARTIST": primary,
|
||||
"ALBUM": t["album"],
|
||||
"TITLE": t["title"],
|
||||
"TRACKNUMBER": str(t["track_number"]),
|
||||
"DISCNUMBER": str(t["disc_number"]),
|
||||
}
|
||||
if t["year"]:
|
||||
pairs["DATE"] = t["year"]
|
||||
# Strip "release identification" tags from MB-tagged Soulseek uploads.
|
||||
# Navidrome's BFR scanner uses these (alongside MB IDs) to differentiate
|
||||
# releases of the same album, so leftover RELEASE*/BARCODE/CATALOGNUMBER/
|
||||
# LABEL/MEDIA tags on one track will split it off into its own album card.
|
||||
# Saw this with Danny Brown's Atrocity Exhibition where Pneumonia had MB
|
||||
# IDs AND a CATALOGNUMBER and split off twice in succession before we
|
||||
# broadened the strip list. Keep this list in sync with strip-mb-tags.sh.
|
||||
stale_mb_tags = [
|
||||
# MusicBrainz IDs
|
||||
"MUSICBRAINZ_ALBUMID", "MUSICBRAINZ_RELEASEGROUPID",
|
||||
"MUSICBRAINZ_RELEASETRACKID", "MUSICBRAINZ_ALBUMARTISTID",
|
||||
"MUSICBRAINZ_ARTISTID", "MUSICBRAINZ_ALBUMSTATUS",
|
||||
"MUSICBRAINZ_ALBUMTYPE", "MUSICBRAINZ_TRACKID", "MUSICBRAINZ_WORKID",
|
||||
# Release identification
|
||||
"RELEASESTATUS", "RELEASETYPE", "RELEASECOUNTRY",
|
||||
"BARCODE", "CATALOGNUMBER", "LABEL", "PUBLISHER",
|
||||
"MEDIA", "ORIGINALDATE", "ASIN", "ISRC", "SCRIPT", "LANGUAGE",
|
||||
# Duplicate / sort variants of artist fields (we use canonical ARTIST/ALBUMARTIST)
|
||||
"ALBUM ARTIST", "ALBUM_ARTIST",
|
||||
"ALBUMARTIST_CREDIT", "ALBUMARTISTSORT", "ALBUMARTISTS",
|
||||
"ALBUM_ARTISTS", "ALBUMARTISTS_CREDIT", "ALBUMARTISTS_SORT",
|
||||
"ARTIST_CREDIT", "ARTISTSORT", "ARTISTS",
|
||||
"ARTISTS_CREDIT", "ARTISTS_SORT", "COMPOSERSORT",
|
||||
# Misc cruft
|
||||
"ACOUSTID_ID", "ACOUSTID_FINGERPRINT", "COMPILATION",
|
||||
"YEAR", # duplicate of DATE
|
||||
"DISCSUBTITLE", "DISCC", "TOTALDISCS", "TRACKC", "TOTALTRACKS",
|
||||
# Kept in sync with mb-tags.sh (was missing these four)
|
||||
"MUSICBRAINZ_ALBUMCOMMENT", "ALBUMCOMMENT",
|
||||
"MUSICBRAINZ_DISCID", "MUSICBRAINZ_TRMID",
|
||||
]
|
||||
# One metaflac invocation: options are applied in order and the file is
|
||||
# rewritten once, so a failure leaves the original tags intact — the old
|
||||
# per-tag loop (~65 subprocesses) could die between remove and set, leaving
|
||||
# a file stripped of ARTIST/TITLE.
|
||||
args = ["metaflac"]
|
||||
args += [f"--remove-tag={tag}" for tag in list(pairs) + stale_mb_tags]
|
||||
args += [f"--set-tag={tag}={val}" for tag, val in pairs.items()]
|
||||
args.append(path)
|
||||
subprocess.run(args, check=True, stderr=log)
|
||||
|
||||
|
||||
def write_mp3_tags(path: str, t: dict, log) -> None:
|
||||
artists_joined = "; ".join(t["artists"])
|
||||
args = [
|
||||
"id3v2",
|
||||
"--TPE1", artists_joined,
|
||||
"--TPE2", t["artists"][0],
|
||||
"--TALB", t["album"],
|
||||
"--TIT2", t["title"],
|
||||
"--TRCK", str(t["track_number"]),
|
||||
"--TPOS", str(t["disc_number"]),
|
||||
]
|
||||
if t["year"]:
|
||||
args += ["--TYER", t["year"]]
|
||||
args.append(path)
|
||||
subprocess.run(args, check=True, stderr=log)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit(f"Usage: {sys.argv[0]} <playlist_url> <scan_dir|->")
|
||||
url, target = sys.argv[1], sys.argv[2]
|
||||
cid = os.environ.get("SPOTIFY_CLIENT_ID")
|
||||
csec = os.environ.get("SPOTIFY_CLIENT_SECRET")
|
||||
if not cid or not csec:
|
||||
sys.exit("SPOTIFY_CLIENT_ID and SPOTIFY_CLIENT_SECRET must be set")
|
||||
|
||||
if target == "-":
|
||||
files_iter = (Path(line.strip()) for line in sys.stdin if line.strip())
|
||||
else:
|
||||
files_iter = Path(target).rglob("*")
|
||||
|
||||
print(f"[spotify-retag] Fetching playlist {url}")
|
||||
token = get_token(cid, csec)
|
||||
tracks = fetch_playlist(url, token)
|
||||
print(f"[spotify-retag] Got {len(tracks)} tracks from Spotify")
|
||||
|
||||
idx = index_by_title(tracks)
|
||||
matched = unmatched = failed = 0
|
||||
unmatched_names: list[str] = []
|
||||
|
||||
for f in sorted(files_iter):
|
||||
if not f.is_file() or f.suffix.lower() not in (".flac", ".mp3"):
|
||||
continue
|
||||
match = match_file(f, idx)
|
||||
if not match:
|
||||
unmatched += 1
|
||||
unmatched_names.append(f.name)
|
||||
continue
|
||||
try:
|
||||
if f.suffix.lower() == ".flac":
|
||||
write_flac_tags(str(f), match, sys.stderr)
|
||||
else:
|
||||
write_mp3_tags(str(f), match, sys.stderr)
|
||||
matched += 1
|
||||
print(f" OK {match['artists'][0]} — {match['title']} ({f.name})")
|
||||
except subprocess.CalledProcessError as e:
|
||||
failed += 1
|
||||
print(f" FAIL {f.name}: {e}", file=sys.stderr)
|
||||
|
||||
print(f"\n[spotify-retag] {matched} matched, {unmatched} unmatched, {failed} failed")
|
||||
for u in unmatched_names[:25]:
|
||||
print(f" unmatched: {u}")
|
||||
if len(unmatched_names) > 25:
|
||||
print(f" ...and {len(unmatched_names) - 25} more")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
# Bulk-strip "release identification" tags from every FLAC in the library.
|
||||
# These come from MB-tagged Soulseek uploads; Navidrome's BFR scanner uses them
|
||||
# to differentiate releases of the same album, which causes one MB-tagged track
|
||||
# to split off into its own album card alongside the non-MB-tagged siblings.
|
||||
#
|
||||
# Run after a bulk import of MB-tagged files, or whenever you see album cards
|
||||
# split in Navidrome with one track in one card and the rest in another.
|
||||
#
|
||||
# The tag list and per-file strip logic live in lib/mb-tags.sh, which is also
|
||||
# sourced by import-track.sh to strip MB tags on every manual import.
|
||||
|
||||
set -u
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/mb-strip-$(date +%Y%m%d-%H%M%S).log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
|
||||
echo "[$(date -Iseconds)] Stripping release-identification tags from FLAC files in ${MUSIC_DATA_DIR:-/data/music}/Library" | tee -a "$LOG"
|
||||
|
||||
COUNT=$(strip_mb_tags ${MUSIC_DATA_DIR:-/data/music}/Library "$LOG")
|
||||
|
||||
echo "[$(date -Iseconds)] Stripped tags from $COUNT FLAC files" | tee -a "$LOG"
|
||||
echo "[$(date -Iseconds)] Syncing beets DB..." | tee -a "$LOG"
|
||||
beet update 2>&1 | tail -5 >> "$LOG"
|
||||
echo "[$(date -Iseconds)] Done. Log: $LOG" | tee -a "$LOG"
|
||||
echo "[$(date -Iseconds)] Trigger a Navidrome scan now to re-evaluate album grouping" | tee -a "$LOG"
|
||||
Executable
+178
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
strip-watermark-art.py — find embedded cover art that appears across multiple
|
||||
unrelated albums (common Soulseek-uploader watermarks like djsoundtop.com,
|
||||
iptorrents.com, torrentday.com, electronicfresh.com, etc.). Strip those
|
||||
images from the offending files; beets' fetchart can re-fetch real cover
|
||||
art on the next pass.
|
||||
|
||||
Heuristic: a real cover.jpg appears in tracks of one (albumartist, album)
|
||||
pair. A watermark image appears across many. Anything where the same image
|
||||
hash appears in >= --threshold distinct albums is flagged.
|
||||
|
||||
Handles both FLAC (via metaflac) and MP3 (via mutagen, apt-installed
|
||||
python3-mutagen).
|
||||
|
||||
Usage:
|
||||
strip-watermark-art.py # dry run
|
||||
strip-watermark-art.py --apply # strip suspicious art
|
||||
strip-watermark-art.py --threshold 5 # tune (default 3)
|
||||
"""
|
||||
import sys, os, hashlib, subprocess, tempfile, argparse
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError, APIC
|
||||
from mutagen.mp3 import MP3
|
||||
|
||||
LIBRARY = f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library"
|
||||
|
||||
|
||||
def build_album_index():
|
||||
"""Single beets call → {path: (albumartist, album)} dict. beets runs
|
||||
in-process in this same container, so paths need no translation."""
|
||||
print("[strip-art] loading beets album index...")
|
||||
r = subprocess.run(
|
||||
["beet", "ls", "-f", "$path‖$albumartist‖$album"],
|
||||
capture_output=True, text=True, timeout=120
|
||||
)
|
||||
idx = {}
|
||||
for line in r.stdout.splitlines():
|
||||
parts = line.split("‖", 2)
|
||||
if len(parts) != 3:
|
||||
continue
|
||||
path, aa, al = parts
|
||||
idx[path] = (aa, al)
|
||||
print(f"[strip-art] indexed {len(idx)} library tracks")
|
||||
return idx
|
||||
|
||||
|
||||
def extract_flac_picture(flac_path):
|
||||
"""Returns (sha256_hash, size) of the first embedded picture, or None."""
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".pic") as tf:
|
||||
tmp = tf.name
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["metaflac", f"--export-picture-to={tmp}", flac_path],
|
||||
capture_output=True, timeout=15
|
||||
)
|
||||
if r.returncode != 0 or not os.path.exists(tmp):
|
||||
return None
|
||||
sz = os.path.getsize(tmp)
|
||||
if sz == 0:
|
||||
return None
|
||||
with open(tmp, "rb") as f:
|
||||
return hashlib.sha256(f.read()).hexdigest(), sz
|
||||
finally:
|
||||
try: os.unlink(tmp)
|
||||
except FileNotFoundError: pass
|
||||
|
||||
|
||||
def extract_mp3_picture(mp3_path):
|
||||
"""Returns (sha256_hash, size) of the first APIC payload, or None."""
|
||||
try:
|
||||
tags = ID3(mp3_path)
|
||||
except (ID3NoHeaderError, Exception):
|
||||
return None
|
||||
for k in tags.keys():
|
||||
if not k.startswith("APIC"):
|
||||
continue
|
||||
frame = tags[k]
|
||||
data = frame.data
|
||||
if data and len(data) > 100:
|
||||
return hashlib.sha256(data).hexdigest(), len(data)
|
||||
return None
|
||||
|
||||
|
||||
def strip_mp3_pictures(mp3_path):
|
||||
"""Remove all APIC frames from an MP3. Returns True on success."""
|
||||
try:
|
||||
tags = ID3(mp3_path)
|
||||
keys = [k for k in tags.keys() if k.startswith("APIC")]
|
||||
for k in keys:
|
||||
del tags[k]
|
||||
tags.save(mp3_path)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true",
|
||||
help="Actually strip suspicious art (default: dry run)")
|
||||
ap.add_argument("--threshold", type=int, default=3,
|
||||
help="Min distinct albums sharing an image to flag it (default 3)")
|
||||
args = ap.parse_args()
|
||||
|
||||
album_idx = build_album_index()
|
||||
|
||||
# hash -> {albums: set of (aa, al), files: [paths], size: int}
|
||||
images = defaultdict(lambda: {"albums": set(), "files": [], "size": 0})
|
||||
audio_files = (
|
||||
list(Path(LIBRARY).rglob("*.flac")) +
|
||||
list(Path(LIBRARY).rglob("*.mp3"))
|
||||
)
|
||||
print(f"[strip-art] scanning {len(audio_files)} audio files for embedded art...")
|
||||
|
||||
for i, p in enumerate(audio_files, 1):
|
||||
if i % 200 == 0:
|
||||
print(f" ...{i}/{len(audio_files)}", flush=True)
|
||||
sfx = p.suffix.lower()
|
||||
if sfx == ".flac":
|
||||
res = extract_flac_picture(str(p))
|
||||
elif sfx == ".mp3":
|
||||
res = extract_mp3_picture(str(p))
|
||||
else:
|
||||
continue
|
||||
if not res:
|
||||
continue
|
||||
h, sz = res
|
||||
aa, al = album_idx.get(str(p), ("", ""))
|
||||
images[h]["albums"].add((aa, al))
|
||||
images[h]["files"].append(str(p))
|
||||
images[h]["size"] = sz
|
||||
|
||||
suspicious = {h: info for h, info in images.items()
|
||||
if len(info["albums"]) >= args.threshold}
|
||||
|
||||
if not suspicious:
|
||||
print(f"\n[strip-art] no images shared across >= {args.threshold} distinct albums.")
|
||||
print("[strip-art] If watermarks remain, lower --threshold or check MP3s manually.")
|
||||
return 0
|
||||
|
||||
print(f"\n[strip-art] {len(suspicious)} suspicious image(s):\n")
|
||||
for h, info in sorted(suspicious.items(), key=lambda kv: -len(kv[1]["files"])):
|
||||
print(f" {h[:16]}… {info['size']} bytes "
|
||||
f"in {len(info['files'])} files across {len(info['albums'])} albums:")
|
||||
for aa, al in sorted(info["albums"])[:6]:
|
||||
print(f" {aa or '?'} / {al or '?'}")
|
||||
if len(info["albums"]) > 6:
|
||||
print(f" ...and {len(info['albums']) - 6} more")
|
||||
print()
|
||||
|
||||
if not args.apply:
|
||||
n = sum(len(s["files"]) for s in suspicious.values())
|
||||
print(f"[strip-art] DRY RUN — re-run with --apply to strip from {n} FLAC files")
|
||||
return 0
|
||||
|
||||
stripped = 0
|
||||
for info in suspicious.values():
|
||||
for f in info["files"]:
|
||||
if f.lower().endswith(".flac"):
|
||||
r = subprocess.run(
|
||||
["metaflac", "--remove", "--block-type=PICTURE", f],
|
||||
capture_output=True
|
||||
)
|
||||
if r.returncode == 0:
|
||||
stripped += 1
|
||||
elif f.lower().endswith(".mp3"):
|
||||
if strip_mp3_pictures(f):
|
||||
stripped += 1
|
||||
|
||||
print(f"\n[strip-art] stripped picture blocks from {stripped} files")
|
||||
print("[strip-art] next: re-fetch real cover art with:")
|
||||
print(" beet fetchart")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+440
@@ -0,0 +1,440 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
sync-bandcamp.py — download new items from a Bandcamp purchase history.
|
||||
|
||||
Reads a Netscape-format cookies file, queries Bandcamp's collection API to
|
||||
find every item the authenticated user has purchased, then downloads each
|
||||
new item (one not in the state file yet) as a ZIP, extracts the audio files,
|
||||
and records the item in the state file.
|
||||
|
||||
Stdlib only — no third-party deps.
|
||||
|
||||
Usage:
|
||||
sync-bandcamp.py <username> <cookies_file> <state_file> <output_dir> [<format_pref>]
|
||||
|
||||
format_pref is a space-separated preference list (default: "flac mp3-320").
|
||||
First match wins.
|
||||
|
||||
Exit code 0 even if some items failed (we want cron to keep trying); details
|
||||
are logged. Exit code !=0 only on auth/setup problems where retrying without
|
||||
human intervention is pointless.
|
||||
"""
|
||||
import sys, os, re, json, time, html, subprocess
|
||||
import urllib.request, urllib.parse
|
||||
from http.cookiejar import MozillaCookieJar
|
||||
from pathlib import Path
|
||||
import zipfile, io
|
||||
import traceback
|
||||
|
||||
|
||||
COLLECTION_ITEMS_URL = "https://bandcamp.com/api/fancollection/1/collection_items"
|
||||
UA = "Mozilla/5.0 (X11; Linux x86_64) sync-bandcamp/1.0"
|
||||
REQUEST_TIMEOUT = 60
|
||||
DOWNLOAD_TIMEOUT = 600 # ZIPs can be large
|
||||
SLEEP_BETWEEN = 2.0 # be polite
|
||||
|
||||
|
||||
def opener_for(jar):
|
||||
return urllib.request.build_opener(
|
||||
urllib.request.HTTPCookieProcessor(jar)
|
||||
)
|
||||
|
||||
|
||||
def http_get(url, jar, timeout=REQUEST_TIMEOUT):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||||
return opener_for(jar).open(req, timeout=timeout)
|
||||
|
||||
|
||||
def http_post_json(url, body, jar, timeout=REQUEST_TIMEOUT):
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=json.dumps(body).encode(),
|
||||
headers={
|
||||
"User-Agent": UA,
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
"Referer": "https://bandcamp.com/",
|
||||
},
|
||||
)
|
||||
return opener_for(jar).open(req, timeout=timeout)
|
||||
|
||||
|
||||
def get_fan_id(username, jar):
|
||||
"""The fan_id is inside the data-blob JSON on the user's collection page.
|
||||
Bandcamp HTML-escapes the JSON inside the attribute, so we have to
|
||||
unescape before matching."""
|
||||
resp = http_get(f"https://bandcamp.com/{username}", jar)
|
||||
page = resp.read().decode("utf-8", errors="replace")
|
||||
page_unescaped = html.unescape(page)
|
||||
m = re.search(r'"fan_id"\s*:\s*"?(\d+)', page_unescaped)
|
||||
if not m:
|
||||
sys.stderr.write("[bandcamp] Could not find fan_id on profile page.\n")
|
||||
sys.stderr.write("[bandcamp] Likely your cookies file is missing or expired,\n")
|
||||
sys.stderr.write("[bandcamp] or the username in config.env is wrong.\n")
|
||||
sys.exit(2)
|
||||
return int(m.group(1))
|
||||
|
||||
|
||||
def fetch_collection(fan_id, jar):
|
||||
"""Page through the collection. Returns (items, redownload_urls).
|
||||
|
||||
items: list of dicts (Bandcamp's per-item JSON)
|
||||
redownload_urls: dict like {"p12345": "https://bandcamp.com/download?...", ...}
|
||||
keyed by <type-letter><sale_item_id>.
|
||||
"""
|
||||
all_items = []
|
||||
all_redownload = {}
|
||||
older_than_token = "9999999999::a::" # paginate from newest
|
||||
|
||||
while True:
|
||||
body = {
|
||||
"fan_id": fan_id,
|
||||
"older_than_token": older_than_token,
|
||||
"count": 100,
|
||||
}
|
||||
resp = http_post_json(COLLECTION_ITEMS_URL, body, jar)
|
||||
data = json.loads(resp.read())
|
||||
|
||||
page_items = data.get("items", [])
|
||||
page_redownload = data.get("redownload_urls", {}) or {}
|
||||
all_items.extend(page_items)
|
||||
all_redownload.update(page_redownload)
|
||||
|
||||
if not data.get("more_available") or not page_items:
|
||||
break
|
||||
|
||||
last_token = data.get("last_token")
|
||||
if not last_token or last_token == older_than_token:
|
||||
break
|
||||
older_than_token = last_token
|
||||
|
||||
return all_items, all_redownload
|
||||
|
||||
|
||||
def parse_data_blob(html_text):
|
||||
"""Bandcamp embeds a JSON 'data-blob' attribute on <div id="pagedata">.
|
||||
Extract and decode it."""
|
||||
m = re.search(r'id="pagedata"[^>]*data-blob="([^"]+)"', html_text)
|
||||
if not m:
|
||||
return None
|
||||
raw = m.group(1)
|
||||
raw = html.unescape(raw)
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def get_format_url(redownload_url, format_pref, jar):
|
||||
"""Load the download landing page and return the signed URL for the
|
||||
preferred format. Returns (format_chosen, url) or (None, None)."""
|
||||
resp = http_get(redownload_url, jar, timeout=REQUEST_TIMEOUT)
|
||||
page = resp.read().decode("utf-8", errors="replace")
|
||||
blob = parse_data_blob(page)
|
||||
if not blob:
|
||||
return None, None
|
||||
|
||||
download_items = blob.get("download_items") or []
|
||||
if not download_items:
|
||||
return None, None
|
||||
|
||||
item = download_items[0]
|
||||
downloads = item.get("downloads") or {}
|
||||
for fmt in format_pref:
|
||||
if fmt in downloads and downloads[fmt].get("url"):
|
||||
return fmt, downloads[fmt]["url"]
|
||||
return None, None
|
||||
|
||||
|
||||
def stat_url_until_ready(stat_url, jar, max_wait_sec=120):
|
||||
"""Bandcamp sometimes returns a 'stat' URL that has to be polled until
|
||||
download is signed. Returns the final download_url, or None if it
|
||||
never becomes ready."""
|
||||
deadline = time.time() + max_wait_sec
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
# The stat endpoint expects ?.rand=<num>
|
||||
url = stat_url + ("&" if "?" in stat_url else "?") + ".rand=" + str(int(time.time() * 1000))
|
||||
try:
|
||||
resp = http_get(url, jar, timeout=30)
|
||||
text = resp.read().decode("utf-8", errors="replace")
|
||||
# Strip JSONP wrapper if present
|
||||
text = re.sub(r"^\s*[A-Za-z_]\w*\s*\(", "", text)
|
||||
text = re.sub(r"\)\s*;?\s*$", "", text)
|
||||
data = json.loads(text)
|
||||
for k, v in (data.get("download_url_stat") or {}).items():
|
||||
last = v
|
||||
if last and last.get("download_ready"):
|
||||
return last.get("download_url")
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(3)
|
||||
return None
|
||||
|
||||
|
||||
def download_payload(url, jar):
|
||||
"""Returns (bytes, suggested_filename) — Bandcamp returns either a ZIP
|
||||
(for albums / multi-track purchases) or a single audio file (for single-
|
||||
track purchases). The Content-Disposition header tells us which."""
|
||||
resp = http_get(url, jar, timeout=DOWNLOAD_TIMEOUT)
|
||||
cd = resp.headers.get("Content-Disposition", "")
|
||||
m = re.search(r'filename\*?=(?:UTF-8\'\')?"?([^";]+)"?', cd)
|
||||
suggested = m.group(1) if m else ""
|
||||
suggested = urllib.parse.unquote(suggested).strip()
|
||||
return resp.read(), suggested
|
||||
|
||||
|
||||
def safe_name(s):
|
||||
"""Make a string safe for filesystem use (and not too long)."""
|
||||
s = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", s)
|
||||
s = s.strip().rstrip(".")
|
||||
return s[:80] if len(s) > 80 else s
|
||||
|
||||
|
||||
_AUDIO_EXTS = (".flac", ".mp3", ".m4a", ".aac", ".ogg", ".oga", ".alac",
|
||||
".wav", ".aiff", ".aif")
|
||||
_IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp")
|
||||
|
||||
# --- Buy-link tagging -------------------------------------------------------
|
||||
# We stamp the release's Bandcamp page into the file so AzuraCast can show a
|
||||
# "Buy" button on ephemeral.club. AzuraCast only auto-assigns custom fields
|
||||
# from its *known* tag enum, so a bespoke "BUY_URL" tag would be ignored
|
||||
# (it lands in extraTags). Instead we use the "Commercial Information" tag —
|
||||
# ID3 frame WCOM, getID3 key `commercial_information`, semantically "a webpage
|
||||
# with information such as where the album can be bought". AzuraCast's custom
|
||||
# field `buy_url` has auto_assign=commercial_information, so this surfaces in
|
||||
# now_playing.song.custom_fields.buy_url. See README "Buy links" section.
|
||||
BUY_URL_TAG = "COMMERCIAL_INFORMATION" # FLAC Vorbis comment name
|
||||
|
||||
|
||||
def tag_buy_url(filepath, url):
|
||||
"""Best-effort: stamp the purchasable URL into the file's commercial-
|
||||
information tag. Never fails the download over a tagging error."""
|
||||
if not url:
|
||||
return
|
||||
low = filepath.lower()
|
||||
try:
|
||||
if low.endswith(".flac"):
|
||||
subprocess.run(["metaflac", f"--remove-tag={BUY_URL_TAG}", filepath],
|
||||
capture_output=True, check=False)
|
||||
subprocess.run(["metaflac", f"--set-tag={BUY_URL_TAG}={url}", filepath],
|
||||
capture_output=True, check=False)
|
||||
elif low.endswith(".mp3"):
|
||||
# WCOM = "Commercial information" URL link frame.
|
||||
from mutagen.id3 import ID3, WCOM, ID3NoHeaderError
|
||||
try:
|
||||
tags = ID3(filepath)
|
||||
except ID3NoHeaderError:
|
||||
tags = ID3()
|
||||
tags.delall("WCOM")
|
||||
tags.add(WCOM(url=url))
|
||||
tags.save(filepath)
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"[bandcamp] WARN: buy_url tag failed on {filepath}: {e}\n")
|
||||
|
||||
|
||||
def save_payload(payload, suggested_name, dest_dir, item_title=None, buy_url=None):
|
||||
"""Save a Bandcamp download — either ZIP (extract audio) or raw audio
|
||||
(write directly). For raw single-track audio, ensure an ALBUM tag is set
|
||||
so the file doesn't land as [Unknown Album] in Navidrome. Each audio file
|
||||
is stamped with the release's buy_url (see tag_buy_url).
|
||||
|
||||
Returns count of audio files written."""
|
||||
# Detect ZIP by magic bytes
|
||||
is_zip = payload[:4] == b"PK\x03\x04" or payload[:4] == b"PK\x05\x06"
|
||||
if is_zip:
|
||||
count = 0
|
||||
with zipfile.ZipFile(io.BytesIO(payload)) as z:
|
||||
for n in z.namelist():
|
||||
low = n.lower()
|
||||
if low.endswith(_AUDIO_EXTS):
|
||||
outp = z.extract(n, dest_dir)
|
||||
tag_buy_url(outp, buy_url)
|
||||
count += 1
|
||||
elif low.endswith(_IMAGE_EXTS):
|
||||
z.extract(n, dest_dir) # cover art for beets fetchart
|
||||
return count
|
||||
|
||||
# Raw audio file (single-track purchase) — Bandcamp serves these without
|
||||
# any ALBUM tag, so we tag them ourselves before they hit the pipeline.
|
||||
name = suggested_name or "track"
|
||||
low = name.lower()
|
||||
if not low.endswith(_AUDIO_EXTS):
|
||||
if payload[:4] == b"fLaC":
|
||||
name = (name or "track") + ".flac"
|
||||
elif payload[:3] == b"ID3" or payload[:2] == b"\xff\xfb":
|
||||
name = (name or "track") + ".mp3"
|
||||
else:
|
||||
return 0 # unknown format
|
||||
name = safe_name(name)
|
||||
out = os.path.join(dest_dir, name)
|
||||
with open(out, "wb") as f:
|
||||
f.write(payload)
|
||||
|
||||
# Set ALBUM = "{item_title} - Single" if the file has no ALBUM tag.
|
||||
# Matches Spotify/Apple Music's convention for single-track releases.
|
||||
if item_title:
|
||||
_ensure_album_tag(out, f"{item_title} - Single")
|
||||
tag_buy_url(out, buy_url)
|
||||
return 1
|
||||
|
||||
|
||||
def _ensure_album_tag(filepath, fallback_album):
|
||||
"""If the file has no ALBUM tag, set it. No-op if one already exists."""
|
||||
low = filepath.lower()
|
||||
try:
|
||||
if low.endswith(".flac"):
|
||||
r = subprocess.run(
|
||||
["metaflac", "--show-tag=ALBUM", filepath],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
if "ALBUM=" in r.stdout:
|
||||
return
|
||||
subprocess.run(
|
||||
["metaflac", f"--set-tag=ALBUM={fallback_album}", filepath],
|
||||
check=True, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
elif low.endswith(".mp3"):
|
||||
r = subprocess.run(
|
||||
["id3v2", "-l", filepath],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
if re.search(r"^TALB\b", r.stdout, re.MULTILINE):
|
||||
return
|
||||
subprocess.run(
|
||||
["id3v2", "--TALB", fallback_album, filepath],
|
||||
check=True, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
pass # best-effort; don't fail the whole download over this
|
||||
|
||||
|
||||
def load_state(path):
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except json.JSONDecodeError:
|
||||
sys.stderr.write(f"[bandcamp] WARN: {path} is corrupt; starting fresh\n")
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(path, state):
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(state, f, indent=2, sort_keys=True)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 5:
|
||||
sys.exit(f"Usage: {sys.argv[0]} <username> <cookies_file> <state_file> <output_dir> [<format_pref>]")
|
||||
|
||||
username = sys.argv[1]
|
||||
cookies_file = sys.argv[2]
|
||||
state_file = sys.argv[3]
|
||||
output_dir = sys.argv[4]
|
||||
format_pref = sys.argv[5].split() if len(sys.argv) > 5 else ["flac", "mp3-320"]
|
||||
|
||||
if not os.path.exists(cookies_file):
|
||||
sys.stderr.write(f"[bandcamp] ERROR: cookies file not found: {cookies_file}\n")
|
||||
sys.stderr.write("[bandcamp] Export cookies for bandcamp.com from a logged-in browser tab\n")
|
||||
sys.stderr.write("[bandcamp] (e.g. via the 'Get cookies.txt LOCALLY' extension) and save here.\n")
|
||||
sys.exit(2)
|
||||
|
||||
jar = MozillaCookieJar(cookies_file)
|
||||
try:
|
||||
jar.load(ignore_discard=True, ignore_expires=True)
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"[bandcamp] ERROR: cookies file unreadable: {e}\n")
|
||||
sys.exit(2)
|
||||
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
state = load_state(state_file)
|
||||
|
||||
print(f"[bandcamp] auth as {username!r}, format pref={format_pref}")
|
||||
fan_id = get_fan_id(username, jar)
|
||||
print(f"[bandcamp] fan_id={fan_id}")
|
||||
|
||||
items, redownload = fetch_collection(fan_id, jar)
|
||||
print(f"[bandcamp] collection has {len(items)} items, {len(redownload)} download URLs")
|
||||
|
||||
new_count = skipped = failed = 0
|
||||
for item in items:
|
||||
sale_id = item.get("sale_item_id")
|
||||
if not sale_id:
|
||||
continue
|
||||
sale_id_str = str(sale_id)
|
||||
if sale_id_str in state:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
band = item.get("band_name") or "Unknown Artist"
|
||||
title = item.get("item_title") or f"item-{item.get('item_id')}"
|
||||
buy_url = item.get("item_url") # the release's Bandcamp page (purchasable)
|
||||
item_type = item.get("item_type", "package") # "track" or "package" (album)
|
||||
type_prefix = "p" if item_type in ("package", "album") else "t"
|
||||
url_key = f"{type_prefix}{sale_id}"
|
||||
|
||||
download_landing = redownload.get(url_key)
|
||||
if not download_landing:
|
||||
# Sometimes the key uses the OTHER prefix
|
||||
alt_prefix = "t" if type_prefix == "p" else "p"
|
||||
download_landing = redownload.get(f"{alt_prefix}{sale_id}")
|
||||
if not download_landing:
|
||||
print(f"[bandcamp] NO_URL {band} — {title}")
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
fmt, signed_url = get_format_url(download_landing, format_pref, jar)
|
||||
if not signed_url:
|
||||
print(f"[bandcamp] NO_FORMAT {band} — {title} (preferred: {format_pref})")
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# Some signed_urls are direct, some are "stat" URLs that need polling
|
||||
actual_url = signed_url
|
||||
if "/statdownload/" in signed_url:
|
||||
polled = stat_url_until_ready(signed_url, jar)
|
||||
if polled:
|
||||
actual_url = polled
|
||||
else:
|
||||
print(f"[bandcamp] STAT_TIMEOUT {band} — {title}")
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
payload, suggested = download_payload(actual_url, jar)
|
||||
dest = Path(output_dir) / safe_name(band) / safe_name(title)
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
n = save_payload(payload, suggested, str(dest), item_title=title,
|
||||
buy_url=buy_url)
|
||||
if n == 0:
|
||||
print(f"[bandcamp] EMPTY_PAYLOAD {band} — {title} (got {len(payload)} bytes, type unknown)")
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
state[sale_id_str] = {
|
||||
"downloaded_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"band": band,
|
||||
"title": title,
|
||||
"format": fmt,
|
||||
"files": n,
|
||||
"path": str(dest),
|
||||
"buy_url": buy_url,
|
||||
}
|
||||
save_state(state_file, state) # save after each — survives interruption
|
||||
print(f"[bandcamp] OK ({fmt}, {n} files) {band} — {title}")
|
||||
new_count += 1
|
||||
time.sleep(SLEEP_BETWEEN)
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"[bandcamp] FAIL {band} — {title}: {e}\n")
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
failed += 1
|
||||
|
||||
print(f"\n[bandcamp] === {new_count} new, {skipped} already had, {failed} failed ===")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+122
@@ -0,0 +1,122 @@
|
||||
#!/bin/bash
|
||||
# Wrapper: sync new Bandcamp purchases, then push them through the standard
|
||||
# import-track.sh pipeline so they end up in the Navidrome library tagged
|
||||
# the same way as everything else.
|
||||
#
|
||||
# Reads $ALEMBIC_CONFIG_DIR/pipeline/bandcamp/config.env for username, format pref, etc.
|
||||
# Run as root (cron). Logs to ${ALEMBIC_CONFIG_DIR:-/config}/logs/bandcamp-YYYYMMDD.log.
|
||||
|
||||
set -u
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
export PATH
|
||||
|
||||
CONFIG="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/bandcamp/config.env"
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/bandcamp-$(date +%Y%m%d).log
|
||||
SYNC_PY=${PIPELINE_DIR:-/app/pipeline}/lib/sync-bandcamp.py
|
||||
IMPORT_ME=${MUSIC_DATA_DIR:-/data/music}/import-me
|
||||
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
log() { echo "[$(date -Iseconds)] $*" | tee -a "$LOG"; }
|
||||
|
||||
if [[ ! -f "$CONFIG" ]]; then
|
||||
log "ERROR: $CONFIG not found"
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG"
|
||||
|
||||
if [[ ! -f "$BANDCAMP_COOKIES" ]]; then
|
||||
log "ERROR: cookies file missing at $BANDCAMP_COOKIES"
|
||||
log " Export Bandcamp cookies in Netscape format (Get cookies.txt LOCALLY"
|
||||
log " extension, while logged into bandcamp.com), save to that path."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Refuse to run against a dead/empty share: mkdir -p below would otherwise
|
||||
# silently create shadow directories inside the container, downloads would
|
||||
# land there invisibly, and state.json would mark those purchases as done —
|
||||
# permanently skipping them after the mount comes back. `mountpoint -q` (the
|
||||
# original host-side check) doesn't apply to a bind-mounted container path;
|
||||
# a canary file + minimum artist-dir count is the container-compatible
|
||||
# equivalent (same pattern as navidrome-scan.sh's share-health gate).
|
||||
LIB="${MUSIC_DATA_DIR:-/data/music}/Library"
|
||||
CANARY="$LIB/.navidrome-canary"
|
||||
MIN_ARTIST_DIRS=500
|
||||
if [[ ! -r "$CANARY" ]]; then
|
||||
log "ERROR: canary $CANARY missing/unreadable — share looks unhealthy — aborting sync"
|
||||
exit 1
|
||||
fi
|
||||
n=$(find "$LIB" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l)
|
||||
if [[ "$n" -lt "$MIN_ARTIST_DIRS" ]]; then
|
||||
log "ERROR: only $n artist dirs under $LIB (< $MIN_ARTIST_DIRS) — library looks truncated — aborting sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$BANDCAMP_STAGING" "$IMPORT_ME"
|
||||
|
||||
log "=== Bandcamp sync starting ==="
|
||||
log "user=$BANDCAMP_USERNAME staging=$BANDCAMP_STAGING format=$BANDCAMP_FORMAT_PREF"
|
||||
|
||||
if ! python3 "$SYNC_PY" \
|
||||
"$BANDCAMP_USERNAME" \
|
||||
"$BANDCAMP_COOKIES" \
|
||||
"$BANDCAMP_STATE" \
|
||||
"$BANDCAMP_STAGING" \
|
||||
"$BANDCAMP_FORMAT_PREF" >> "$LOG" 2>&1; then
|
||||
log "WARN: sync-bandcamp.py exited non-zero (likely auth issue or network)"
|
||||
fi
|
||||
|
||||
# Anything new? If staging dir has audio, hand it off to import-track.sh.
|
||||
NEW_COUNT=$(find "$BANDCAMP_STAGING" -type f \( -iname "*.flac" -o -iname "*.mp3" -o -iname "*.wav" -o -iname "*.m4a" \) 2>/dev/null | wc -l)
|
||||
if [[ "$NEW_COUNT" -eq 0 ]]; then
|
||||
log "No new files to import"
|
||||
log "=== Bandcamp sync done (no-op) ==="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "$NEW_COUNT new audio file(s) staged; moving to $IMPORT_ME for import"
|
||||
|
||||
# Move everything from staging into import-me. Preserve the band/album folder
|
||||
# structure since import-track.sh handles directories fine.
|
||||
shopt -s dotglob nullglob
|
||||
for entry in "$BANDCAMP_STAGING"/*; do
|
||||
[[ -e "$entry" ]] || continue
|
||||
mv "$entry" "$IMPORT_ME/" 2>>"$LOG"
|
||||
done
|
||||
shopt -u dotglob nullglob
|
||||
|
||||
# Hand off to the standard manual-import pipeline. No playlist tag — Bandcamp
|
||||
# purchases aren't part of any Spotify playlist. The import-track.sh guard,
|
||||
# albumartist fallback, beets import, and Navidrome scan all kick in normally.
|
||||
log "Calling import-track.sh"
|
||||
if ${PIPELINE_DIR:-/app/pipeline}/bin/import-track.sh >> "$LOG" 2>&1; then
|
||||
log "import-track.sh OK"
|
||||
else
|
||||
log "WARN: import-track.sh exited non-zero (exit $?)"
|
||||
fi
|
||||
|
||||
# Safety net: if anything is stranded in ${MUSIC_DATA_DIR:-/data/music}/downloads (rare with
|
||||
# duplicate_action=keep set in beets config, but possible for files beets
|
||||
# couldn't process), find each one's library counterpart and replace if the
|
||||
# new file is higher quality, OR import as new if there's no counterpart.
|
||||
log "Running replace-with-better safety pass on /downloads stragglers"
|
||||
if ${PIPELINE_DIR:-/app/pipeline}/lib/replace-with-better.sh --apply >> "$LOG" 2>&1; then
|
||||
log "replace-with-better OK"
|
||||
else
|
||||
log "WARN: replace-with-better.sh exited non-zero (exit $?)"
|
||||
fi
|
||||
|
||||
# Now dedupe across the library. With duplicate_action=keep beets imported
|
||||
# every Bandcamp file even when it conflicts with an existing track at the
|
||||
# same path (it creates `.1.ext` siblings). dedup-library.sh's policy is
|
||||
# "FLAC > MP3, then largest file" — Bandcamp version wins, soulseek version
|
||||
# is removed from both the beets DB and disk.
|
||||
log "Running dedup pass (Bandcamp FLAC wins over older Soulseek copies)"
|
||||
if ${PIPELINE_DIR:-/app/pipeline}/lib/dedup-library.sh --apply >> "$LOG" 2>&1; then
|
||||
log "dedup OK"
|
||||
else
|
||||
log "WARN: dedup-library.sh exited non-zero (exit $?)"
|
||||
fi
|
||||
|
||||
log "=== Bandcamp sync done ==="
|
||||
exit 0
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/bin/bash
|
||||
# ${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
#
|
||||
# Shared helper sourced by run-playlist.sh and import-track.sh.
|
||||
# Moves audio files with missing ARTIST or TITLE tags into a quarantine dir,
|
||||
# so they never reach beets and rot the library as "[Unknown Album]".
|
||||
#
|
||||
# Usage: quarantine_untagged <scan_dir> <quarantine_dir> <log_file>
|
||||
# Returns: number of files moved (via stdout)
|
||||
|
||||
# Separator pattern matching Navidrome's artist split list in navidrome.toml.
|
||||
# Used by set_albumartist_fallback to extract the primary artist from a multi-
|
||||
# artist string. Keep these two lists in sync.
|
||||
#
|
||||
# Only unambiguous separators here. ", " and " & " are NOT included because they
|
||||
# appear inside real band names ("Tyler, The Creator", "Simon & Garfunkel",
|
||||
# "piri & tommy"). For collab strings using those separators, ALBUMARTIST falls
|
||||
# back to the full string — Navidrome will show one combined artist for the album,
|
||||
# which is the lesser of two evils vs. corrupting real band names.
|
||||
_PRIMARY_ARTIST_SEP_PATTERN=' / | feat\. | feat | ft\. | ft |; '
|
||||
|
||||
# For each audio file under $scan_dir that has no ALBUMARTIST tag, set
|
||||
# ALBUMARTIST to the primary artist (everything before the first separator
|
||||
# in the ARTIST tag). Prevents Navidrome from coining ghost album-artists
|
||||
# like "Fred Again, The Blessed Madonna" when a track is a collab.
|
||||
#
|
||||
# Usage: set_albumartist_fallback <scan_dir> <log_file>
|
||||
# Returns: number of files updated (via stdout)
|
||||
set_albumartist_fallback() {
|
||||
local scan_dir="$1"
|
||||
local logfile="$2"
|
||||
local count=0
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
case "${file##*.}" in
|
||||
flac|FLAC)
|
||||
local cur_aa artist primary
|
||||
cur_aa=$(metaflac --show-tag=ALBUMARTIST "$file" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
[[ -n "$cur_aa" ]] && continue
|
||||
artist=$(metaflac --show-tag=ARTIST "$file" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
[[ -z "$artist" ]] && continue
|
||||
primary=$(echo "$artist" | sed -E "s#(${_PRIMARY_ARTIST_SEP_PATTERN}).*##")
|
||||
metaflac --set-tag="ALBUMARTIST=${primary}" "$file" 2>>"$logfile"
|
||||
count=$((count + 1))
|
||||
;;
|
||||
mp3|MP3)
|
||||
local cur_aa artist primary
|
||||
cur_aa=$(id3v2 -l "$file" 2>/dev/null | sed -n 's/^TPE2[^:]*: //p' | head -1)
|
||||
[[ -n "$cur_aa" ]] && continue
|
||||
artist=$(id3v2 -l "$file" 2>/dev/null | sed -n 's/^TPE1[^:]*: //p' | head -1)
|
||||
[[ -z "$artist" ]] && continue
|
||||
primary=$(echo "$artist" | sed -E "s#(${_PRIMARY_ARTIST_SEP_PATTERN}).*##")
|
||||
id3v2 --TPE2 "$primary" "$file" 2>>"$logfile"
|
||||
count=$((count + 1))
|
||||
;;
|
||||
esac
|
||||
done < <(find "$scan_dir" -type f \( -iname "*.flac" -o -iname "*.mp3" \) -print0 2>/dev/null)
|
||||
|
||||
echo "$count"
|
||||
}
|
||||
|
||||
quarantine_untagged() {
|
||||
local scan_dir="$1"
|
||||
local quarantine_dir="$2"
|
||||
local logfile="$3"
|
||||
local moved=0
|
||||
|
||||
mkdir -p "$quarantine_dir"
|
||||
|
||||
_has_flac_tags() {
|
||||
local artist title
|
||||
artist=$(metaflac --show-tag=ARTIST "$1" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
title=$(metaflac --show-tag=TITLE "$1" 2>/dev/null | sed -n 's/^[^=]*=//p' | head -1)
|
||||
[[ -n "$artist" && -n "$title" ]]
|
||||
}
|
||||
|
||||
_has_mp3_tags() {
|
||||
local artist title
|
||||
artist=$(id3v2 -l "$1" 2>/dev/null | sed -n 's/^TPE1[^:]*: //p' | head -1)
|
||||
title=$(id3v2 -l "$1" 2>/dev/null | sed -n 's/^TIT2[^:]*: //p' | head -1)
|
||||
[[ -n "$artist" && -n "$title" ]]
|
||||
}
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
local ok=1
|
||||
case "${file##*.}" in
|
||||
flac|FLAC) _has_flac_tags "$file" || ok=0 ;;
|
||||
mp3|MP3) _has_mp3_tags "$file" || ok=0 ;;
|
||||
*) ok=0 ;; # unknown format — beets can't handle it anyway
|
||||
esac
|
||||
if [[ $ok -eq 0 ]]; then
|
||||
echo "[$(date -Iseconds)] QUARANTINE: missing artist/title — $(basename "$file")" >> "$logfile"
|
||||
mv -n "$file" "$quarantine_dir/" 2>>"$logfile"
|
||||
moved=$((moved + 1))
|
||||
fi
|
||||
done < <(find "$scan_dir" -type f \( -iname "*.flac" -o -iname "*.mp3" -o -iname "*.m4a" \) -print0 2>/dev/null)
|
||||
|
||||
echo "$moved"
|
||||
}
|
||||
Executable
+180
@@ -0,0 +1,180 @@
|
||||
#!/bin/bash
|
||||
# upgrade-mp3-to-flac.sh — for every MP3 currently in the library, ask sldl
|
||||
# to find a FLAC version of the same track. If found, replace the MP3.
|
||||
# If not found, the MP3 stays (we don't lose anything).
|
||||
#
|
||||
# Strategy:
|
||||
# 1. Build a CSV from beets with one row per MP3 track.
|
||||
# 2. Run sldl with --format flac (HARD format requirement, not preferred —
|
||||
# sldl will skip any track where no FLAC is available rather than
|
||||
# falling back to MP3).
|
||||
# 3. For each FLAC that arrives in the staging dropbox, run
|
||||
# replace-with-better.sh which already knows the FLAC > MP3 rule.
|
||||
#
|
||||
# Conservative — if Soulseek peers are slow that day or the FLAC isn't
|
||||
# available, we leave the MP3 alone. Re-runs are cheap; sldl's m3u skip
|
||||
# logic means it won't re-search tracks it has already FLAC-found.
|
||||
#
|
||||
# Usage:
|
||||
# upgrade-mp3-to-flac.sh # run the scanner end-to-end
|
||||
# upgrade-mp3-to-flac.sh --csv-only # just write the CSV; don't run sldl
|
||||
|
||||
set -u
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
export PATH
|
||||
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/tag-guard.sh
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/prep-audio.sh
|
||||
# shellcheck source=${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
source ${PIPELINE_DIR:-/app/pipeline}/lib/mb-tags.sh
|
||||
|
||||
CSV_ONLY=0
|
||||
[[ "${1:-}" == "--csv-only" ]] && CSV_ONLY=1
|
||||
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/upgrade-mp3-$(date +%Y%m%d-%H%M%S).log
|
||||
STAGING_HOST=${MUSIC_DATA_DIR:-/data/music}/sldl-dropbox/_upgrade
|
||||
QUARANTINE_DIR=${MUSIC_DATA_DIR:-/data/music}/Songs/untagged
|
||||
CSV_HOST=${ALEMBIC_CONFIG_DIR:-/config}/pipeline/_upgrade.csv
|
||||
SLDL_BIN=${PIPELINE_DIR:-/app/pipeline}/sldl
|
||||
|
||||
mkdir -p "$(dirname "$LOG")" "$STAGING_HOST"
|
||||
log() { echo "[$(date -Iseconds)] $*" | tee -a "$LOG"; }
|
||||
|
||||
log "=== MP3 upgrade scan starting ==="
|
||||
|
||||
# Build CSV from beets. sldl's CSV format: comma-separated with header
|
||||
# Artist, Title, Album, Length (sec), URI (optional). We omit URI; sldl
|
||||
# falls back to a Soulseek text search when no URI is given.
|
||||
log "Querying beets for MP3 tracks..."
|
||||
beet ls -f '$artist|$title|$album|$length' 'format:mp3' 2>/dev/null > /tmp/mp3-rows.txt
|
||||
COUNT=$(wc -l < /tmp/mp3-rows.txt)
|
||||
log " $COUNT MP3 tracks to attempt"
|
||||
|
||||
# Convert | to , and quote fields that contain commas
|
||||
{
|
||||
echo "Artist,Title,Album,Length"
|
||||
awk -F'|' 'BEGIN{OFS=""} {
|
||||
for (i=1; i<=NF; i++) {
|
||||
g = $i; gsub(/"/, "\"\"", g)
|
||||
if (g ~ /[,"]/) g = "\"" g "\""
|
||||
$i = g
|
||||
}
|
||||
print $1 "," $2 "," $3 "," $4
|
||||
}' /tmp/mp3-rows.txt
|
||||
} > "$CSV_HOST"
|
||||
|
||||
log " CSV written: $CSV_HOST ($(wc -l < "$CSV_HOST") lines incl. header)"
|
||||
|
||||
if [[ $CSV_ONLY -eq 1 ]]; then
|
||||
log "--csv-only flag — exiting before sldl run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Pull credentials from any playlist conf (they're all the same)
|
||||
SOULSEEK_USER=$(sed -n 's/^user *= *//p' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/y2k.conf | tr -d ' ')
|
||||
SOULSEEK_PASS=$(sed -n 's/^pass *= *//p' ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/y2k.conf | tr -d ' ')
|
||||
|
||||
log "Running sldl with --format flac (HARD requirement, no MP3/WAV fallback)"
|
||||
"$SLDL_BIN" \
|
||||
"$CSV_HOST" \
|
||||
--input-type csv \
|
||||
--user "$SOULSEEK_USER" \
|
||||
--pass "$SOULSEEK_PASS" \
|
||||
--path "$STAGING_HOST" \
|
||||
--format flac \
|
||||
--pref-min-bitrate 320 \
|
||||
--concurrent-downloads 8 \
|
||||
--search-timeout 30000 \
|
||||
--strict-title \
|
||||
--strict-artist \
|
||||
--no-browse-folder \
|
||||
--no-skip-existing \
|
||||
--verbose \
|
||||
>> "$LOG" 2>&1
|
||||
SLDL_EXIT=$?
|
||||
log "sldl exit code: $SLDL_EXIT"
|
||||
|
||||
# Guard: this script is FLAC-upgrade-only. If sldl ever drops anything else
|
||||
# in staging (WAV/MP3/etc.), purge it so the import-leftover fallback in
|
||||
# replace-with-better.sh doesn't pull non-FLACs into the library.
|
||||
NON_FLAC=$(find "$STAGING_HOST" -type f ! -iname "*.flac" ! -name "*.incomplete" 2>/dev/null)
|
||||
if [[ -n "$NON_FLAC" ]]; then
|
||||
log "Purging non-FLAC files from staging:"
|
||||
echo "$NON_FLAC" | tee -a "$LOG"
|
||||
echo "$NON_FLAC" | xargs -r rm -f
|
||||
fi
|
||||
|
||||
NEW_FLAC_COUNT=$(find "$STAGING_HOST" -type f -iname "*.flac" 2>/dev/null | wc -l)
|
||||
log "Staging has $NEW_FLAC_COUNT FLAC file(s) ready for upgrade"
|
||||
|
||||
if [[ "$NEW_FLAC_COUNT" -eq 0 ]]; then
|
||||
log "Nothing to upgrade. Done."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# === Treat staging files like new playlist downloads ===
|
||||
# Same hygiene + prep as run-playlist.sh, applied to /sldl-dropbox/_upgrade
|
||||
# before any of these files enter the library.
|
||||
|
||||
# Quarantine FLACs with missing ARTIST/TITLE — same rule as playlist runs.
|
||||
# Prevents the __.wav-style "untagged files in /Library/ root" failure mode.
|
||||
QUARANTINED=$(quarantine_untagged "$STAGING_HOST" "$QUARANTINE_DIR" "$LOG" || echo 0)
|
||||
[[ "$QUARANTINED" -gt 0 ]] && log "Quarantined $QUARANTINED file(s) to $QUARANTINE_DIR"
|
||||
|
||||
# Recount post-quarantine
|
||||
NEW_FLAC_COUNT=$(find "$STAGING_HOST" -type f -iname "*.flac" 2>/dev/null | wc -l)
|
||||
[[ "$NEW_FLAC_COUNT" -eq 0 ]] && { log "Nothing left after quarantine. Done."; exit 0; }
|
||||
|
||||
# Strip MB IDs (prevents Navidrome album fragmentation). Scoped to staging:
|
||||
# the library-wide strip-mb-tags.sh runs weekly anyway and wouldn't touch
|
||||
# staging FLACs (they aren't in /Library/ yet).
|
||||
MB_STRIPPED=$(strip_mb_tags "$STAGING_HOST" "$LOG" || echo 0)
|
||||
log "Stripped MB tags from $MB_STRIPPED file(s) in staging"
|
||||
|
||||
# ALBUMARTIST fallback — safety net for no-match leftovers whose uploader
|
||||
# didn't set ALBUMARTIST. For matched files this is overwritten by the
|
||||
# tag-copy step inside replace-with-better.sh (canonical MP3 tags win).
|
||||
AA_SET=$(set_albumartist_fallback "$STAGING_HOST" "$LOG" || echo 0)
|
||||
[[ "$AA_SET" -gt 0 ]] && log "Set ALBUMARTIST fallback on $AA_SET file(s)"
|
||||
|
||||
# ReplayGain + autocue tagging — same as run-playlist.sh.
|
||||
log "Pre-tagging with ReplayGain + autocue"
|
||||
PREPPED=$(prep_audio "$STAGING_HOST" "$LOG" || echo 0)
|
||||
log "Pre-tagged $PREPPED file(s)"
|
||||
|
||||
# Use replace-with-better.sh against the staging dir. --copy-tags-from-existing
|
||||
# makes it copy canonical tags (incl. GROUPING) from each matched MP3 onto
|
||||
# the replacing FLAC before the swap, so playlist M3Us still resolve and the
|
||||
# FLAC inherits spotify-retag's canonical metadata.
|
||||
log "Running replace-with-better.sh on staging (--copy-tags-from-existing)"
|
||||
${PIPELINE_DIR:-/app/pipeline}/lib/replace-with-better.sh --apply --copy-tags-from-existing "$STAGING_HOST" >> "$LOG" 2>&1
|
||||
RBE_EXIT=$?
|
||||
log "replace-with-better exit: $RBE_EXIT"
|
||||
|
||||
# Regenerate playlist M3Us. Upgraded tracks have new paths (FLAC vs MP3) but
|
||||
# their GROUPING tags were carried over via --copy-tags-from-existing, so the
|
||||
# beet-driven regen below resolves them to the new FLAC paths.
|
||||
log "Regenerating playlist M3Us"
|
||||
PLAYLISTS_DIR=${MUSIC_DATA_DIR:-/data/music}/playlists
|
||||
mkdir -p "$PLAYLISTS_DIR"
|
||||
M3U_COUNT=0
|
||||
while IFS= read -r playlist; do
|
||||
[[ -z "$playlist" ]] && continue
|
||||
M3U_OUT="${PLAYLISTS_DIR}/${playlist}.m3u8"
|
||||
BEET_OUTPUT=$(beet ls -f '$path' "grouping:${playlist}" 2>>"$LOG" || true)
|
||||
TRACK_COUNT=$(echo -n "$BEET_OUTPUT" | grep -c '^')
|
||||
[[ "$TRACK_COUNT" -gt 0 ]] || continue
|
||||
{ echo "#EXTM3U"; echo "$BEET_OUTPUT"; } > "$M3U_OUT"
|
||||
log " $playlist.m3u8 — $TRACK_COUNT tracks"
|
||||
M3U_COUNT=$((M3U_COUNT + 1))
|
||||
done < <(beet ls -f '$grouping' 2>/dev/null | sort -u | grep -v '^$')
|
||||
log "Regenerated $M3U_COUNT M3U file(s)"
|
||||
|
||||
# Cleanup staging
|
||||
log "Cleaning up empty staging dir"
|
||||
find "$STAGING_HOST" -type f -delete 2>>"$LOG" || true
|
||||
find "$STAGING_HOST" -mindepth 1 -type d -empty -delete 2>>"$LOG" || true
|
||||
|
||||
log "=== MP3 upgrade scan done ==="
|
||||
Reference in New Issue
Block a user