andrew
068e7e9534
scrub-watermark-text.py crashed on every run (missing import os). strip-watermark-art.py was starved every Sunday by lock contention from strip-mb-tags' growing runtime -- moved it to 01:00 so the rest of the weekly chain has room. enrich-buy-url.py's summary line counted matches found, not successful writes, so a run where every metaflac write failed (root-owned files) still reported "N tagged". pipeline-status.sh now tells a 403 from Qobuz (Akamai/CDN block) apart from a 401 (actually expired token) -- re-exporting the token does nothing for the former. Also fixes the dedup review table's "Caught by" badge overflowing into the Keep column's format pill.
556 lines
24 KiB
Python
Executable File
556 lines
24 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
enrich-buy-url.py — best-effort buy links for non-purchase (Soulseek) tracks.
|
|
|
|
For every library FLAC that has no buy link yet, look the track up by
|
|
artist + title and, on an EXACT match, stamp the store page into the
|
|
COMMERCIAL_INFORMATION tag (same field the Bandcamp-purchase pipeline uses,
|
|
which AzuraCast maps to the buy_url custom field — see README "Buy links").
|
|
|
|
Source cascade (configurable), ordered by quality / DRM-freedom / artist
|
|
payout: Bandcamp first (lossless, best payout), then Qobuz (DRM-free hi-res),
|
|
then the iTunes Search API as the last-resort fallback. EXACT matching only —
|
|
exact artist (primary artist; collabs allowed as a leading match) AND exact
|
|
title — to keep wrong links out of the files, per the err-toward-false-negatives
|
|
policy.
|
|
|
|
Qobuz needs a logged-in session: it reads a user_auth_token (from --qobuz-token,
|
|
$QOBUZ_AUTH_TOKEN, or $ALEMBIC_CONFIG_DIR/pipeline/qobuz/token — grab it from the Web Player's
|
|
X-User-Auth-Token request header) and an app_id (auto-scraped from the Web Player
|
|
bundle, or --qobuz-app-id / $QOBUZ_APP_ID / $ALEMBIC_CONFIG_DIR/pipeline/qobuz/app_id). With no
|
|
token, Qobuz is silently dropped from the cascade and the run falls through to
|
|
the next source.
|
|
|
|
Resumable: files that already carry a buy link are skipped, so a re-run only
|
|
processes new/untagged tracks. Safe to wire into cron after the nightly
|
|
playlist runs.
|
|
|
|
Stdlib only. Dry-run by default.
|
|
|
|
Usage:
|
|
enrich-buy-url.py [--library DIR] [--apply] [--force]
|
|
[--sources bandcamp,qobuz,itunes] [--limit N]
|
|
[--bc-sleep S] [--qb-sleep S] [--it-sleep S]
|
|
[--qobuz-token TOKEN] [--qobuz-app-id ID]
|
|
[--azuracast-key ID:SECRET] [--azuracast-base URL] [--station N]
|
|
"""
|
|
import sys, os, re, json, time, argparse, subprocess
|
|
import urllib.request, urllib.parse
|
|
from pathlib import Path
|
|
|
|
UA = "Mozilla/5.0 (X11; Linux x86_64) enrich-buy-url/1.0"
|
|
BUY_URL_TAG = "COMMERCIAL_INFORMATION"
|
|
# Marker (unix timestamp) stamped on a FLAC when a buy-link lookup found
|
|
# nothing, so daily runs don't re-query the whole no-match backlog every time.
|
|
# Retried after the window below; --force ignores it.
|
|
BUY_URL_TRIED_TAG = "BUY_URL_TRIED"
|
|
_TRIED_RETRY_DAYS = 30
|
|
|
|
|
|
def _tried_recently(ts_str):
|
|
try:
|
|
return (time.time() - float(ts_str)) < _TRIED_RETRY_DAYS * 86400
|
|
except (TypeError, ValueError):
|
|
return False
|
|
ITUNES_SEARCH = "https://itunes.apple.com/search"
|
|
BC_AUTOCOMPLETE = "https://bandcamp.com/api/bcsearch_public_api/1/autocomplete_elastic"
|
|
QOBUZ_SEARCH = "https://www.qobuz.com/api.json/0.2/track/search"
|
|
QOBUZ_ALBUM_GET = "https://www.qobuz.com/api.json/0.2/album/get"
|
|
QOBUZ_LOGIN_PAGE = "https://play.qobuz.com/login"
|
|
_ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
|
|
QOBUZ_TOKEN_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/token"
|
|
QOBUZ_APPID_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/app_id"
|
|
QOBUZ_REGION_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/region"
|
|
QOBUZ_DEFAULT_REGION = "ca-en"
|
|
|
|
# Populated once by resolve_qobuz() during startup; None disables the source.
|
|
# Carries {token, app_id, region}.
|
|
QOBUZ_CREDS = None
|
|
|
|
|
|
# ---- matching --------------------------------------------------------------
|
|
def norm(s):
|
|
s = (s or "").lower()
|
|
s = re.sub(r"\((feat|ft|featuring|with)\.?[^)]*\)", "", s)
|
|
s = re.sub(r"\b(feat|ft|featuring)\.?\s.*$", "", s)
|
|
s = re.sub(r"[^a-z0-9]+", " ", s)
|
|
return re.sub(r"\s+", " ", s).strip()
|
|
|
|
|
|
def artist_ok(query, result):
|
|
"""Exact, or the queried (primary) artist leads a collab credit."""
|
|
nq, nr = norm(query), norm(result)
|
|
if not nq or not nr:
|
|
return False
|
|
return nr == nq or nr.startswith(nq + " ") or nq.startswith(nr + " ")
|
|
|
|
|
|
def title_ok(query, result):
|
|
return norm(query) == norm(result) and norm(query) != ""
|
|
|
|
|
|
def url_source(url):
|
|
"""Best-effort: which store an already-stored buy link came from. Used by
|
|
--upgrade-from to spot links worth re-evaluating against higher-priority
|
|
sources. Unknown/white-label domains are treated as Bandcamp (its custom
|
|
domains, e.g. music.monstercat.com, are the only ones we can't fingerprint)."""
|
|
u = (url or "").lower()
|
|
if "apple.com" in u or "itunes" in u:
|
|
return "itunes"
|
|
if "qobuz.com" in u:
|
|
return "qobuz"
|
|
return "bandcamp"
|
|
|
|
|
|
# ---- sources ---------------------------------------------------------------
|
|
def _open(req, timeout):
|
|
"""One retry with backoff on throttle/temporary errors (polite + better yield)."""
|
|
for attempt in range(2):
|
|
try:
|
|
return urllib.request.urlopen(req, timeout=timeout).read()
|
|
except urllib.error.HTTPError as e:
|
|
if e.code in (429, 500, 502, 503) and attempt == 0:
|
|
time.sleep(5)
|
|
continue
|
|
raise
|
|
|
|
|
|
def _get(url, timeout=30):
|
|
return _open(urllib.request.Request(url, headers={"User-Agent": UA}), timeout)
|
|
|
|
|
|
def _post(url, body, timeout=30):
|
|
req = urllib.request.Request(
|
|
url, data=json.dumps(body).encode(),
|
|
headers={"User-Agent": UA, "Content-Type": "application/json"})
|
|
return _open(req, timeout)
|
|
|
|
|
|
def lookup_bandcamp(artist, title):
|
|
"""Return a track-page URL on an exact match, else None."""
|
|
try:
|
|
raw = _post(BC_AUTOCOMPLETE, {
|
|
"search_text": f"{artist} {title}",
|
|
"search_filter": "", "full_page": False, "fan_id": None})
|
|
results = (json.loads(raw).get("auto") or {}).get("results", [])
|
|
except Exception:
|
|
return None
|
|
for r in results:
|
|
if r.get("type") != "t": # tracks only (exact title)
|
|
continue
|
|
if artist_ok(artist, r.get("band_name", "")) and title_ok(title, r.get("name", "")):
|
|
return r.get("item_url_path") or r.get("url")
|
|
return None
|
|
|
|
|
|
def lookup_itunes(artist, title):
|
|
"""Return a store URL on an exact match (prefer buyable), else None."""
|
|
try:
|
|
url = ITUNES_SEARCH + "?" + urllib.parse.urlencode(
|
|
{"term": f"{artist} {title}", "entity": "song", "limit": 8})
|
|
results = json.loads(_get(url)).get("results", [])
|
|
except Exception:
|
|
return None
|
|
fallback = None
|
|
for r in results:
|
|
if artist_ok(artist, r.get("artistName", "")) and title_ok(title, r.get("trackName", "")):
|
|
link = r.get("trackViewUrl")
|
|
if not link:
|
|
continue
|
|
if r.get("trackPrice"): # buyable wins
|
|
return link
|
|
fallback = fallback or link
|
|
return fallback
|
|
|
|
|
|
def scrape_qobuz_app_id(log):
|
|
"""Pull the production app_id out of the Web Player bundle. Best-effort:
|
|
returns None if the page/bundle layout changed."""
|
|
try:
|
|
page = _get(QOBUZ_LOGIN_PAGE, timeout=30).decode("utf-8", "replace")
|
|
m = re.search(r"/resources/[\w.\-]+/bundle\.js", page)
|
|
if not m:
|
|
log(" [qobuz] could not locate bundle.js on the login page")
|
|
return None
|
|
bundle = _get("https://play.qobuz.com" + m.group(0), timeout=90).decode("utf-8", "replace")
|
|
m = re.search(r'production:\{api:\{appId:"(\d+)"', bundle)
|
|
if not m:
|
|
log(" [qobuz] could not find appId in bundle.js")
|
|
return None
|
|
return m.group(1)
|
|
except Exception as e:
|
|
log(f" [qobuz] app_id scrape failed: {e}")
|
|
return None
|
|
|
|
|
|
def resolve_qobuz(args, log):
|
|
"""Resolve {token, app_id, region} for the Qobuz source, or None to disable
|
|
it. Token: --qobuz-token, $QOBUZ_AUTH_TOKEN, then the keyfile. app_id: flag/
|
|
env/keyfile, else auto-scraped from the Web Player bundle. region: flag/env/
|
|
keyfile, else the default (controls which store the buy link points at)."""
|
|
token = args.qobuz_token or os.environ.get("QOBUZ_AUTH_TOKEN")
|
|
if not token and os.path.exists(QOBUZ_TOKEN_FILE):
|
|
token = Path(QOBUZ_TOKEN_FILE).read_text().strip()
|
|
if not token:
|
|
log(" [qobuz] no user_auth_token (flag/env/keyfile) — dropping Qobuz from cascade")
|
|
return None
|
|
|
|
app_id = args.qobuz_app_id or os.environ.get("QOBUZ_APP_ID")
|
|
if not app_id and os.path.exists(QOBUZ_APPID_FILE):
|
|
app_id = Path(QOBUZ_APPID_FILE).read_text().strip()
|
|
if not app_id:
|
|
app_id = scrape_qobuz_app_id(log)
|
|
if not app_id:
|
|
log(" [qobuz] no app_id (flag/env/keyfile/scrape) — dropping Qobuz from cascade")
|
|
return None
|
|
|
|
region = getattr(args, "qobuz_region", None) or os.environ.get("QOBUZ_REGION")
|
|
if not region and os.path.exists(QOBUZ_REGION_FILE):
|
|
region = Path(QOBUZ_REGION_FILE).read_text().strip()
|
|
region = (region or QOBUZ_DEFAULT_REGION).strip().strip("/")
|
|
return {"token": token.strip(), "app_id": str(app_id).strip(), "region": region}
|
|
|
|
|
|
def qobuz_store_url(album_id):
|
|
"""Resolve an album id to its canonical, regioned *purchase* page
|
|
(www.qobuz.com/<region>/album/<slug>/<id>) via album/get. Returns None if the
|
|
album isn't purchasable or can't be fetched — so the caller falls through to
|
|
the next source rather than stamping a streaming-only link. We deliberately
|
|
do NOT use open.qobuz.com (that's the player/streaming deep link)."""
|
|
if not (QOBUZ_CREDS and album_id):
|
|
return None
|
|
try:
|
|
url = QOBUZ_ALBUM_GET + "?" + urllib.parse.urlencode(
|
|
{"album_id": album_id, "app_id": QOBUZ_CREDS["app_id"]})
|
|
req = urllib.request.Request(url, headers={
|
|
"User-Agent": UA,
|
|
"X-App-Id": QOBUZ_CREDS["app_id"],
|
|
"X-User-Auth-Token": QOBUZ_CREDS["token"]})
|
|
d = json.loads(_open(req, 30))
|
|
except Exception:
|
|
return None
|
|
# album/get returns a region-less relative_url like /album/<slug>/<id>; the
|
|
# bare `url` is locale-pinned to the account default (fr-fr), so we build from
|
|
# relative_url + the configured region instead.
|
|
rel = d.get("relative_url")
|
|
if not d.get("purchasable") or not rel:
|
|
return None
|
|
return f"https://www.qobuz.com/{QOBUZ_CREDS['region']}{rel}"
|
|
|
|
|
|
def lookup_qobuz(artist, title):
|
|
"""Return a Qobuz *purchase* URL on an exact, purchasable match, else None
|
|
(falls through to the next source). Requires QOBUZ_CREDS."""
|
|
if not QOBUZ_CREDS:
|
|
return None
|
|
try:
|
|
url = QOBUZ_SEARCH + "?" + urllib.parse.urlencode(
|
|
{"query": f"{artist} {title}", "limit": 10, "app_id": QOBUZ_CREDS["app_id"]})
|
|
req = urllib.request.Request(url, headers={
|
|
"User-Agent": UA,
|
|
"X-App-Id": QOBUZ_CREDS["app_id"],
|
|
"X-User-Auth-Token": QOBUZ_CREDS["token"]})
|
|
items = (json.loads(_open(req, 30)).get("tracks") or {}).get("items", [])
|
|
except Exception:
|
|
return None
|
|
tried = 0
|
|
for t in items:
|
|
album = t.get("album") or {}
|
|
performer = (t.get("performer") or {}).get("name", "") or \
|
|
(album.get("artist") or {}).get("name", "")
|
|
if not (artist_ok(artist, performer) and title_ok(title, t.get("title", ""))):
|
|
continue
|
|
# Search carries only the album id; album/get yields the purchase page and
|
|
# gates on purchasability. A given track can appear on several releases
|
|
# (single, album, comp) — try the first few for a purchasable one before
|
|
# giving up and falling through to iTunes.
|
|
time.sleep(0.3) # polite: a second call per match
|
|
link = qobuz_store_url(album.get("id"))
|
|
if link:
|
|
return link
|
|
tried += 1
|
|
if tried >= 3:
|
|
break
|
|
return None
|
|
|
|
|
|
SOURCES = {"bandcamp": lookup_bandcamp, "qobuz": lookup_qobuz, "itunes": lookup_itunes}
|
|
|
|
|
|
# ---- tag I/O ---------------------------------------------------------------
|
|
def flac_tag(path, name):
|
|
r = subprocess.run(["metaflac", f"--show-tag={name}", path], capture_output=True, text=True)
|
|
for line in r.stdout.splitlines():
|
|
if "=" in line:
|
|
return line.split("=", 1)[1]
|
|
return ""
|
|
|
|
|
|
def set_flac_tag(path, name, value):
|
|
subprocess.run(["metaflac", f"--remove-tag={name}", path], capture_output=True)
|
|
return subprocess.run(["metaflac", f"--set-tag={name}={value}", path],
|
|
capture_output=True).returncode == 0
|
|
|
|
|
|
# ---- AzuraCast reprocess (same as backfill-buy-url.py) ----------------------
|
|
def az_reprocess(base, key, station, host_paths, library, log):
|
|
if not base:
|
|
log(" no AzuraCast base URL configured — skipping reprocess")
|
|
return
|
|
root = library.rstrip("/") + "/"
|
|
want = {p[len(root):] for p in host_paths if p.startswith(root)}
|
|
headers = {"X-API-Key": key, "Accept": "application/json"}
|
|
|
|
def get(p):
|
|
return json.loads(urllib.request.urlopen(
|
|
urllib.request.Request(base + p, headers=headers), timeout=90).read())
|
|
|
|
uids, page = [], 1
|
|
while True:
|
|
d = get(f"/api/station/{station}/files?per_page=500&page={page}")
|
|
for row in d["rows"]:
|
|
if row["path"] in want:
|
|
uids.append(row["unique_id"])
|
|
if page >= d["total_pages"]:
|
|
break
|
|
page += 1
|
|
if not uids:
|
|
log(" no matching AzuraCast files to reprocess")
|
|
return
|
|
# AzuraCast caps batch size in practice; chunk to be safe.
|
|
for i in range(0, len(uids), 200):
|
|
chunk = uids[i:i + 200]
|
|
req = urllib.request.Request(
|
|
base + f"/api/station/{station}/files/batch",
|
|
data=json.dumps({"do": "reprocess", "files": chunk}).encode(), method="PUT",
|
|
headers={**headers, "Content-Type": "application/json"})
|
|
urllib.request.urlopen(req, timeout=120).read()
|
|
log(f" queued reprocess for {len(uids)} AzuraCast file(s)")
|
|
|
|
|
|
def refresh_qobuz_links(flacs, args, az_key):
|
|
"""Rewrite existing open.qobuz.com (streaming) buy links to the canonical
|
|
www.qobuz.com purchase page, converting by album id (no re-matching). Links
|
|
that aren't purchasable / can't be resolved are left intact and counted."""
|
|
print(f"[refresh-qobuz] mode={'APPLY' if args.apply else 'DRY RUN'} "
|
|
f"region={QOBUZ_CREDS['region']}")
|
|
print(f"[refresh-qobuz] scanning {len(flacs)} FLAC files...\n")
|
|
looked = converted = unconvertible = 0
|
|
touched = []
|
|
for p in flacs:
|
|
sp = str(p)
|
|
existing = flac_tag(sp, BUY_URL_TAG)
|
|
if not existing or "open.qobuz.com" not in existing:
|
|
continue
|
|
m = re.search(r"open\.qobuz\.com/album/([^/?#]+)", existing)
|
|
if not m:
|
|
continue # track-form deep link: leave it
|
|
if args.limit and looked >= args.limit:
|
|
break
|
|
looked += 1
|
|
time.sleep(args.qb_sleep)
|
|
new = qobuz_store_url(m.group(1))
|
|
rel = sp[len(args.library):].lstrip("/")
|
|
if not new or new == existing:
|
|
unconvertible += 1 # not purchasable in-region / gone
|
|
continue
|
|
converted += 1
|
|
print(f" ~ {rel}\n {existing}\n -> {new}")
|
|
if args.apply:
|
|
if set_flac_tag(sp, BUY_URL_TAG, new):
|
|
touched.append(sp)
|
|
else:
|
|
print(" ! metaflac write failed")
|
|
else:
|
|
touched.append(sp)
|
|
|
|
print(f"\n[refresh-qobuz] {looked} open.qobuz links, "
|
|
f"{converted} {'converted' if args.apply else 'would-convert'}, "
|
|
f"{unconvertible} left intact (not purchasable / unresolved)")
|
|
|
|
if args.apply and touched and az_key and args.azuracast_base:
|
|
print("[refresh-qobuz] telling AzuraCast to reprocess touched files...")
|
|
az_reprocess(args.azuracast_base, az_key, args.station,
|
|
touched, args.library, lambda m: print(m))
|
|
elif not args.apply:
|
|
print("[refresh-qobuz] re-run with --apply to write")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--library", default=f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library")
|
|
ap.add_argument("--apply", action="store_true")
|
|
ap.add_argument("--force", action="store_true", help="re-match files that already have a buy link")
|
|
ap.add_argument("--upgrade-from", default="",
|
|
help="comma-separated sources whose existing links should be "
|
|
"re-evaluated and REPLACED if a higher-priority source now "
|
|
"matches (e.g. 'itunes'). Lower-priority matches never "
|
|
"overwrite; a no-match leaves the existing link intact.")
|
|
ap.add_argument("--sources", default="bandcamp,qobuz,itunes",
|
|
help="comma-separated cascade order (subset of: bandcamp, qobuz, itunes)")
|
|
ap.add_argument("--limit", type=int, default=0, help="stop after N lookups (0 = all)")
|
|
ap.add_argument("--bc-sleep", type=float, default=0.7)
|
|
ap.add_argument("--qb-sleep", type=float, default=0.8)
|
|
ap.add_argument("--it-sleep", type=float, default=1.0)
|
|
ap.add_argument("--qobuz-token", default=None,
|
|
help="Qobuz user_auth_token (else $QOBUZ_AUTH_TOKEN or "
|
|
f"{QOBUZ_TOKEN_FILE})")
|
|
ap.add_argument("--qobuz-app-id", default=None,
|
|
help="Qobuz app_id (else $QOBUZ_APP_ID, keyfile, or auto-scrape)")
|
|
ap.add_argument("--qobuz-region", default=None,
|
|
help=f"Qobuz store region for buy links, e.g. ca-en (else "
|
|
f"$QOBUZ_REGION, {QOBUZ_REGION_FILE}, or {QOBUZ_DEFAULT_REGION})")
|
|
ap.add_argument("--refresh-qobuz", action="store_true",
|
|
help="one-shot: rewrite existing open.qobuz.com (streaming) buy "
|
|
"links to the canonical www.qobuz.com purchase page. No "
|
|
"re-matching — converts by album id. Skips links that "
|
|
"aren't purchasable (leaves them intact).")
|
|
ap.add_argument("--azuracast-key", default=None)
|
|
ap.add_argument("--azuracast-base", default=os.environ.get("AZURACAST_BASE", ""))
|
|
ap.add_argument("--station", type=int, default=1)
|
|
args = ap.parse_args()
|
|
|
|
# Resolve the AzuraCast key (for the reprocess trigger) without putting the
|
|
# secret on the command line / in cron: --azuracast-key, then env, then file.
|
|
az_key = args.azuracast_key or os.environ.get("AZURACAST_API_KEY")
|
|
if not az_key:
|
|
keyfile = f"{_ALEMBIC_CONFIG_DIR}/pipeline/azuracast/api_key"
|
|
if os.path.exists(keyfile):
|
|
az_key = Path(keyfile).read_text().strip()
|
|
|
|
# Same cascade for the base URL: --azuracast-base, then env, then the file
|
|
# credential_service renders from Settings -> Credentials -> AzuraCast.
|
|
# The scheduled job (scheduler_service) invokes this script with neither
|
|
# the flag nor the env var set, so the keyfile is the only way it ever
|
|
# gets one.
|
|
if not args.azuracast_base:
|
|
basefile = f"{_ALEMBIC_CONFIG_DIR}/pipeline/azuracast/base_url"
|
|
if os.path.exists(basefile):
|
|
args.azuracast_base = Path(basefile).read_text().strip()
|
|
|
|
cascade = [s.strip() for s in args.sources.split(",") if s.strip() in SOURCES]
|
|
if not cascade:
|
|
sys.exit(f"[enrich] no valid sources in {args.sources!r}")
|
|
sleep_for = {"bandcamp": args.bc_sleep, "qobuz": args.qb_sleep, "itunes": args.it_sleep}
|
|
|
|
# Qobuz needs auth; resolve it now and drop the source if unavailable so the
|
|
# cascade just falls through (err-toward-false-negatives — no guessed links).
|
|
# --refresh-qobuz also needs the creds (to call album/get).
|
|
global QOBUZ_CREDS
|
|
if "qobuz" in cascade or args.refresh_qobuz:
|
|
QOBUZ_CREDS = resolve_qobuz(args, lambda m: print(m))
|
|
if not QOBUZ_CREDS:
|
|
cascade = [s for s in cascade if s != "qobuz"]
|
|
if args.refresh_qobuz:
|
|
sys.exit("[enrich] --refresh-qobuz needs Qobuz creds — aborting")
|
|
|
|
upgrade_from = {s.strip() for s in args.upgrade_from.split(",")
|
|
if s.strip() in SOURCES}
|
|
|
|
flacs = list(Path(args.library).rglob("*.flac"))
|
|
|
|
# --- one-shot migration: open.qobuz.com (streaming) -> purchase page -------
|
|
if args.refresh_qobuz:
|
|
refresh_qobuz_links(flacs, args, az_key)
|
|
return
|
|
|
|
print(f"[enrich] mode={'APPLY' if args.apply else 'DRY RUN'} cascade={cascade}"
|
|
+ (f" upgrade-from={sorted(upgrade_from)}" if upgrade_from else ""))
|
|
print(f"[enrich] {len(flacs)} FLAC files in library\n")
|
|
|
|
looked = found = upgraded = write_failed = 0
|
|
by_source = {s: 0 for s in cascade}
|
|
touched = []
|
|
for p in flacs:
|
|
sp = str(p)
|
|
existing = flac_tag(sp, BUY_URL_TAG)
|
|
# Decide which sources to actually query for this file.
|
|
if existing and not args.force:
|
|
cur = url_source(existing)
|
|
if upgrade_from and cur in upgrade_from:
|
|
# Only try sources ranked ABOVE the current one — a same/lower
|
|
# match must never replace it. No-match leaves the link intact.
|
|
run_sources = cascade[:cascade.index(cur)] if cur in cascade else list(cascade)
|
|
if not run_sources:
|
|
continue
|
|
else:
|
|
continue # already tagged, not up for upgrade
|
|
else:
|
|
# Skip tracks we recently tried and found nothing for, so a daily
|
|
# run doesn't re-query the whole no-match backlog (external, rate-
|
|
# limited lookups) every time -- that churn is what pushed this job
|
|
# past its timeout. --force re-checks everything.
|
|
if not args.force:
|
|
tried = flac_tag(sp, BUY_URL_TRIED_TAG)
|
|
if tried and _tried_recently(tried):
|
|
continue
|
|
run_sources = cascade # untagged, or --force
|
|
artist = flac_tag(sp, "ARTIST").split(";")[0].strip() or flac_tag(sp, "ALBUMARTIST")
|
|
title = flac_tag(sp, "TITLE")
|
|
if not (artist and title):
|
|
continue
|
|
if args.limit and looked >= args.limit:
|
|
break
|
|
looked += 1
|
|
|
|
url = src = None
|
|
for s in run_sources:
|
|
url = SOURCES[s](artist, title)
|
|
time.sleep(sleep_for[s])
|
|
if url:
|
|
src = s
|
|
break
|
|
|
|
rel = sp[len(args.library):].lstrip("/")
|
|
if not url:
|
|
# Record that we checked and found nothing, so the next daily run
|
|
# skips it (until the retry window). Apply mode only.
|
|
if args.apply:
|
|
set_flac_tag(sp, BUY_URL_TRIED_TAG, str(int(time.time())))
|
|
continue
|
|
found += 1
|
|
by_source[src] += 1
|
|
if existing:
|
|
upgraded += 1
|
|
print(f" ~ [{src}] {artist} - {title}\n -> {url}\n (replaces {existing})")
|
|
else:
|
|
print(f" + [{src}] {artist} - {title}\n -> {url}")
|
|
if args.apply:
|
|
if set_flac_tag(sp, BUY_URL_TAG, url):
|
|
touched.append(sp)
|
|
else:
|
|
write_failed += 1
|
|
print(" ! metaflac write failed (permissions? disk full?) -- NOT tagged")
|
|
else:
|
|
touched.append(sp)
|
|
|
|
if found % 50 == 0:
|
|
print(f" ...{found} links from {looked} lookups so far", flush=True)
|
|
|
|
# touched is only appended to on an actual successful write (apply mode)
|
|
# or a would-tag match (dry run) -- len(touched) is what really landed on
|
|
# disk. `found` counts matches regardless of write outcome, so reporting
|
|
# `found` here as "tagged" lied about full success on 2026-07-22, when
|
|
# every write in the run failed (root-owned files under explo/) but the
|
|
# summary line still read "51 tagged".
|
|
print(f"\n[enrich] {looked} lookups, {len(touched)} {'tagged' if args.apply else 'would-tag'}"
|
|
f" ({', '.join(f'{s}={by_source[s]}' for s in cascade)})"
|
|
f" no-match={looked - found}"
|
|
+ (f" upgraded={upgraded}" if upgrade_from else "")
|
|
+ (f" WRITE-FAILED={write_failed}" if write_failed else ""))
|
|
|
|
if args.apply and touched and az_key and args.azuracast_base:
|
|
print("[enrich] telling AzuraCast to reprocess touched files...")
|
|
az_reprocess(args.azuracast_base, az_key, args.station,
|
|
touched, args.library, lambda m: print(m))
|
|
elif args.apply and touched and (not az_key or not args.azuracast_base):
|
|
print("[enrich] no AzuraCast key/base URL (flag/env/keyfile) — tags written; "
|
|
"AzuraCast's periodic media scan will pick them up.")
|
|
elif not args.apply:
|
|
print("[enrich] re-run with --apply to write (key via --azuracast-key, "
|
|
"$AZURACAST_API_KEY, or $ALEMBIC_CONFIG_DIR/pipeline/azuracast/api_key to reprocess)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|