andrew/alembic
1
0
Fork 0
Code Pull Requests Actions Releases 18 Wiki Activity

Stage 0: migrate pipeline scripts from /opt/sldl, scaffold repo

Browse Source 
Moves all ~30 pipeline scripts, configs, and the vendored sldl binary into
this repo (source /opt/sldl left untouched). Removes all docker exec/docker
compose dependencies now that beets and sldl run in-process/as a subprocess
of this container instead of via soulbeet/on-demand sldl containers.
Replaces hardcoded host paths, Navidrome credentials, and Spotify credential
sourcing with env-var-driven paths and shared credential loaders. Adds
Dockerfile, entrypoint.sh, requirements.txt, docker-compose.snippet.yml, and
the initial app DB schema.
This commit is contained in:
andrew
2026-07-08 13:10:55 -06:00
commit 68cb007e4c
48 changed files with 9261 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline/lib/enrich-buy-url.py
Executable
+510
View File
@@ -0,0 +1,510 @@
#!/usr/bin/env python3
"""
enrich-buy-url.py — best-effort buy links for non-purchase (Soulseek) tracks.
For every library FLAC that has no buy link yet, look the track up by
artist + title and, on an EXACT match, stamp the store page into the
COMMERCIAL_INFORMATION tag (same field the Bandcamp-purchase pipeline uses,
which AzuraCast maps to the buy_url custom field — see README "Buy links").
Source cascade (configurable), ordered by quality / DRM-freedom / artist
payout: Bandcamp first (lossless, best payout), then Qobuz (DRM-free hi-res),
then the iTunes Search API as the last-resort fallback. EXACT matching only —
exact artist (primary artist; collabs allowed as a leading match) AND exact
title — to keep wrong links out of the files, per the err-toward-false-negatives
policy.
Qobuz needs a logged-in session: it reads a user_auth_token (from --qobuz-token,
$QOBUZ_AUTH_TOKEN, or $ALEMBIC_CONFIG_DIR/pipeline/qobuz/token — grab it from the Web Player's
X-User-Auth-Token request header) and an app_id (auto-scraped from the Web Player
bundle, or --qobuz-app-id / $QOBUZ_APP_ID / $ALEMBIC_CONFIG_DIR/pipeline/qobuz/app_id). With no
token, Qobuz is silently dropped from the cascade and the run falls through to
the next source.
Resumable: files that already carry a buy link are skipped, so a re-run only
processes new/untagged tracks. Safe to wire into cron after the nightly
playlist runs.
Stdlib only. Dry-run by default.
Usage:
enrich-buy-url.py [--library DIR] [--apply] [--force]
[--sources bandcamp,qobuz,itunes] [--limit N]
[--bc-sleep S] [--qb-sleep S] [--it-sleep S]
[--qobuz-token TOKEN] [--qobuz-app-id ID]
[--azuracast-key ID:SECRET] [--azuracast-base URL] [--station N]
"""
import sys, os, re, json, time, argparse, subprocess
import urllib.request, urllib.parse
from pathlib import Path
UA = "Mozilla/5.0 (X11; Linux x86_64) enrich-buy-url/1.0"
BUY_URL_TAG = "COMMERCIAL_INFORMATION"
ITUNES_SEARCH = "https://itunes.apple.com/search"
BC_AUTOCOMPLETE = "https://bandcamp.com/api/bcsearch_public_api/1/autocomplete_elastic"
QOBUZ_SEARCH = "https://www.qobuz.com/api.json/0.2/track/search"
QOBUZ_ALBUM_GET = "https://www.qobuz.com/api.json/0.2/album/get"
QOBUZ_LOGIN_PAGE = "https://play.qobuz.com/login"
_ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
QOBUZ_TOKEN_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/token"
QOBUZ_APPID_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/app_id"
QOBUZ_REGION_FILE = f"{_ALEMBIC_CONFIG_DIR}/pipeline/qobuz/region"
QOBUZ_DEFAULT_REGION = "ca-en"
# Populated once by resolve_qobuz() during startup; None disables the source.
# Carries {token, app_id, region}.
QOBUZ_CREDS = None
# ---- matching --------------------------------------------------------------
def norm(s):
s = (s or "").lower()
s = re.sub(r"\((feat|ft|featuring|with)\.?[^)]*\)", "", s)
s = re.sub(r"\b(feat|ft|featuring)\.?\s.*$", "", s)
s = re.sub(r"[^a-z0-9]+", " ", s)
return re.sub(r"\s+", " ", s).strip()
def artist_ok(query, result):
"""Exact, or the queried (primary) artist leads a collab credit."""
nq, nr = norm(query), norm(result)
if not nq or not nr:
return False
return nr == nq or nr.startswith(nq + " ") or nq.startswith(nr + " ")
def title_ok(query, result):
return norm(query) == norm(result) and norm(query) != ""
def url_source(url):
"""Best-effort: which store an already-stored buy link came from. Used by
--upgrade-from to spot links worth re-evaluating against higher-priority
sources. Unknown/white-label domains are treated as Bandcamp (its custom
domains, e.g. music.monstercat.com, are the only ones we can't fingerprint)."""
u = (url or "").lower()
if "apple.com" in u or "itunes" in u:
return "itunes"
if "qobuz.com" in u:
return "qobuz"
return "bandcamp"
# ---- sources ---------------------------------------------------------------
def _open(req, timeout):
"""One retry with backoff on throttle/temporary errors (polite + better yield)."""
for attempt in range(2):
try:
return urllib.request.urlopen(req, timeout=timeout).read()
except urllib.error.HTTPError as e:
if e.code in (429, 500, 502, 503) and attempt == 0:
time.sleep(5)
continue
raise
def _get(url, timeout=30):
return _open(urllib.request.Request(url, headers={"User-Agent": UA}), timeout)
def _post(url, body, timeout=30):
req = urllib.request.Request(
url, data=json.dumps(body).encode(),
headers={"User-Agent": UA, "Content-Type": "application/json"})
return _open(req, timeout)
def lookup_bandcamp(artist, title):
"""Return a track-page URL on an exact match, else None."""
try:
raw = _post(BC_AUTOCOMPLETE, {
"search_text": f"{artist} {title}",
"search_filter": "", "full_page": False, "fan_id": None})
results = (json.loads(raw).get("auto") or {}).get("results", [])
except Exception:
return None
for r in results:
if r.get("type") != "t": # tracks only (exact title)
continue
if artist_ok(artist, r.get("band_name", "")) and title_ok(title, r.get("name", "")):
return r.get("item_url_path") or r.get("url")
return None
def lookup_itunes(artist, title):
"""Return a store URL on an exact match (prefer buyable), else None."""
try:
url = ITUNES_SEARCH + "?" + urllib.parse.urlencode(
{"term": f"{artist} {title}", "entity": "song", "limit": 8})
results = json.loads(_get(url)).get("results", [])
except Exception:
return None
fallback = None
for r in results:
if artist_ok(artist, r.get("artistName", "")) and title_ok(title, r.get("trackName", "")):
link = r.get("trackViewUrl")
if not link:
continue
if r.get("trackPrice"): # buyable wins
return link
fallback = fallback or link
return fallback
def scrape_qobuz_app_id(log):
"""Pull the production app_id out of the Web Player bundle. Best-effort:
returns None if the page/bundle layout changed."""
try:
page = _get(QOBUZ_LOGIN_PAGE, timeout=30).decode("utf-8", "replace")
m = re.search(r"/resources/[\w.\-]+/bundle\.js", page)
if not m:
log(" [qobuz] could not locate bundle.js on the login page")
return None
bundle = _get("https://play.qobuz.com" + m.group(0), timeout=90).decode("utf-8", "replace")
m = re.search(r'production:\{api:\{appId:"(\d+)"', bundle)
if not m:
log(" [qobuz] could not find appId in bundle.js")
return None
return m.group(1)
except Exception as e:
log(f" [qobuz] app_id scrape failed: {e}")
return None
def resolve_qobuz(args, log):
"""Resolve {token, app_id, region} for the Qobuz source, or None to disable
it. Token: --qobuz-token, $QOBUZ_AUTH_TOKEN, then the keyfile. app_id: flag/
env/keyfile, else auto-scraped from the Web Player bundle. region: flag/env/
keyfile, else the default (controls which store the buy link points at)."""
token = args.qobuz_token or os.environ.get("QOBUZ_AUTH_TOKEN")
if not token and os.path.exists(QOBUZ_TOKEN_FILE):
token = Path(QOBUZ_TOKEN_FILE).read_text().strip()
if not token:
log(" [qobuz] no user_auth_token (flag/env/keyfile) — dropping Qobuz from cascade")
return None
app_id = args.qobuz_app_id or os.environ.get("QOBUZ_APP_ID")
if not app_id and os.path.exists(QOBUZ_APPID_FILE):
app_id = Path(QOBUZ_APPID_FILE).read_text().strip()
if not app_id:
app_id = scrape_qobuz_app_id(log)
if not app_id:
log(" [qobuz] no app_id (flag/env/keyfile/scrape) — dropping Qobuz from cascade")
return None
region = getattr(args, "qobuz_region", None) or os.environ.get("QOBUZ_REGION")
if not region and os.path.exists(QOBUZ_REGION_FILE):
region = Path(QOBUZ_REGION_FILE).read_text().strip()
region = (region or QOBUZ_DEFAULT_REGION).strip().strip("/")
return {"token": token.strip(), "app_id": str(app_id).strip(), "region": region}
def qobuz_store_url(album_id):
"""Resolve an album id to its canonical, regioned *purchase* page
(www.qobuz.com/<region>/album/<slug>/<id>) via album/get. Returns None if the
album isn't purchasable or can't be fetched — so the caller falls through to
the next source rather than stamping a streaming-only link. We deliberately
do NOT use open.qobuz.com (that's the player/streaming deep link)."""
if not (QOBUZ_CREDS and album_id):
return None
try:
url = QOBUZ_ALBUM_GET + "?" + urllib.parse.urlencode(
{"album_id": album_id, "app_id": QOBUZ_CREDS["app_id"]})
req = urllib.request.Request(url, headers={
"User-Agent": UA,
"X-App-Id": QOBUZ_CREDS["app_id"],
"X-User-Auth-Token": QOBUZ_CREDS["token"]})
d = json.loads(_open(req, 30))
except Exception:
return None
# album/get returns a region-less relative_url like /album/<slug>/<id>; the
# bare `url` is locale-pinned to the account default (fr-fr), so we build from
# relative_url + the configured region instead.
rel = d.get("relative_url")
if not d.get("purchasable") or not rel:
return None
return f"https://www.qobuz.com/{QOBUZ_CREDS['region']}{rel}"
def lookup_qobuz(artist, title):
"""Return a Qobuz *purchase* URL on an exact, purchasable match, else None
(falls through to the next source). Requires QOBUZ_CREDS."""
if not QOBUZ_CREDS:
return None
try:
url = QOBUZ_SEARCH + "?" + urllib.parse.urlencode(
{"query": f"{artist} {title}", "limit": 10, "app_id": QOBUZ_CREDS["app_id"]})
req = urllib.request.Request(url, headers={
"User-Agent": UA,
"X-App-Id": QOBUZ_CREDS["app_id"],
"X-User-Auth-Token": QOBUZ_CREDS["token"]})
items = (json.loads(_open(req, 30)).get("tracks") or {}).get("items", [])
except Exception:
return None
tried = 0
for t in items:
album = t.get("album") or {}
performer = (t.get("performer") or {}).get("name", "") or \
(album.get("artist") or {}).get("name", "")
if not (artist_ok(artist, performer) and title_ok(title, t.get("title", ""))):
continue
# Search carries only the album id; album/get yields the purchase page and
# gates on purchasability. A given track can appear on several releases
# (single, album, comp) — try the first few for a purchasable one before
# giving up and falling through to iTunes.
time.sleep(0.3) # polite: a second call per match
link = qobuz_store_url(album.get("id"))
if link:
return link
tried += 1
if tried >= 3:
break
return None
SOURCES = {"bandcamp": lookup_bandcamp, "qobuz": lookup_qobuz, "itunes": lookup_itunes}
# ---- tag I/O ---------------------------------------------------------------
def flac_tag(path, name):
r = subprocess.run(["metaflac", f"--show-tag={name}", path], capture_output=True, text=True)
for line in r.stdout.splitlines():
if "=" in line:
return line.split("=", 1)[1]
return ""
def set_flac_tag(path, name, value):
subprocess.run(["metaflac", f"--remove-tag={name}", path], capture_output=True)
return subprocess.run(["metaflac", f"--set-tag={name}={value}", path],
capture_output=True).returncode == 0
# ---- AzuraCast reprocess (same as backfill-buy-url.py) ----------------------
def az_reprocess(base, key, station, host_paths, library, log):
root = library.rstrip("/") + "/"
want = {p[len(root):] for p in host_paths if p.startswith(root)}
headers = {"X-API-Key": key, "Accept": "application/json"}
def get(p):
return json.loads(urllib.request.urlopen(
urllib.request.Request(base + p, headers=headers), timeout=90).read())
uids, page = [], 1
while True:
d = get(f"/api/station/{station}/files?per_page=500&page={page}")
for row in d["rows"]:
if row["path"] in want:
uids.append(row["unique_id"])
if page >= d["total_pages"]:
break
page += 1
if not uids:
log(" no matching AzuraCast files to reprocess")
return
# AzuraCast caps batch size in practice; chunk to be safe.
for i in range(0, len(uids), 200):
chunk = uids[i:i + 200]
req = urllib.request.Request(
base + f"/api/station/{station}/files/batch",
data=json.dumps({"do": "reprocess", "files": chunk}).encode(), method="PUT",
headers={**headers, "Content-Type": "application/json"})
urllib.request.urlopen(req, timeout=120).read()
log(f" queued reprocess for {len(uids)} AzuraCast file(s)")
def refresh_qobuz_links(flacs, args, az_key):
"""Rewrite existing open.qobuz.com (streaming) buy links to the canonical
www.qobuz.com purchase page, converting by album id (no re-matching). Links
that aren't purchasable / can't be resolved are left intact and counted."""
print(f"[refresh-qobuz] mode={'APPLY' if args.apply else 'DRY RUN'} "
f"region={QOBUZ_CREDS['region']}")
print(f"[refresh-qobuz] scanning {len(flacs)} FLAC files...\n")
looked = converted = unconvertible = 0
touched = []
for p in flacs:
sp = str(p)
existing = flac_tag(sp, BUY_URL_TAG)
if not existing or "open.qobuz.com" not in existing:
continue
m = re.search(r"open\.qobuz\.com/album/([^/?#]+)", existing)
if not m:
continue # track-form deep link: leave it
if args.limit and looked >= args.limit:
break
looked += 1
time.sleep(args.qb_sleep)
new = qobuz_store_url(m.group(1))
rel = sp[len(args.library):].lstrip("/")
if not new or new == existing:
unconvertible += 1 # not purchasable in-region / gone
continue
converted += 1
print(f" ~ {rel}\n {existing}\n -> {new}")
if args.apply:
if set_flac_tag(sp, BUY_URL_TAG, new):
touched.append(sp)
else:
print(" ! metaflac write failed")
else:
touched.append(sp)
print(f"\n[refresh-qobuz] {looked} open.qobuz links, "
f"{converted} {'converted' if args.apply else 'would-convert'}, "
f"{unconvertible} left intact (not purchasable / unresolved)")
if args.apply and touched and az_key:
print("[refresh-qobuz] telling AzuraCast to reprocess touched files...")
az_reprocess(args.azuracast_base, az_key, args.station,
touched, args.library, lambda m: print(m))
elif not args.apply:
print("[refresh-qobuz] re-run with --apply to write")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--library", default=f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library")
ap.add_argument("--apply", action="store_true")
ap.add_argument("--force", action="store_true", help="re-match files that already have a buy link")
ap.add_argument("--upgrade-from", default="",
help="comma-separated sources whose existing links should be "
"re-evaluated and REPLACED if a higher-priority source now "
"matches (e.g. 'itunes'). Lower-priority matches never "
"overwrite; a no-match leaves the existing link intact.")
ap.add_argument("--sources", default="bandcamp,qobuz,itunes",
help="comma-separated cascade order (subset of: bandcamp, qobuz, itunes)")
ap.add_argument("--limit", type=int, default=0, help="stop after N lookups (0 = all)")
ap.add_argument("--bc-sleep", type=float, default=0.7)
ap.add_argument("--qb-sleep", type=float, default=0.8)
ap.add_argument("--it-sleep", type=float, default=1.0)
ap.add_argument("--qobuz-token", default=None,
help="Qobuz user_auth_token (else $QOBUZ_AUTH_TOKEN or "
f"{QOBUZ_TOKEN_FILE})")
ap.add_argument("--qobuz-app-id", default=None,
help="Qobuz app_id (else $QOBUZ_APP_ID, keyfile, or auto-scrape)")
ap.add_argument("--qobuz-region", default=None,
help=f"Qobuz store region for buy links, e.g. ca-en (else "
f"$QOBUZ_REGION, {QOBUZ_REGION_FILE}, or {QOBUZ_DEFAULT_REGION})")
ap.add_argument("--refresh-qobuz", action="store_true",
help="one-shot: rewrite existing open.qobuz.com (streaming) buy "
"links to the canonical www.qobuz.com purchase page. No "
"re-matching — converts by album id. Skips links that "
"aren't purchasable (leaves them intact).")
ap.add_argument("--azuracast-key", default=None)
ap.add_argument("--azuracast-base", default="https://admin.ephemeral.club")
ap.add_argument("--station", type=int, default=1)
args = ap.parse_args()
# Resolve the AzuraCast key (for the reprocess trigger) without putting the
# secret on the command line / in cron: --azuracast-key, then env, then file.
az_key = args.azuracast_key or os.environ.get("AZURACAST_API_KEY")
if not az_key:
keyfile = f"{_ALEMBIC_CONFIG_DIR}/pipeline/azuracast/api_key"
if os.path.exists(keyfile):
az_key = Path(keyfile).read_text().strip()
cascade = [s.strip() for s in args.sources.split(",") if s.strip() in SOURCES]
if not cascade:
sys.exit(f"[enrich] no valid sources in {args.sources!r}")
sleep_for = {"bandcamp": args.bc_sleep, "qobuz": args.qb_sleep, "itunes": args.it_sleep}
# Qobuz needs auth; resolve it now and drop the source if unavailable so the
# cascade just falls through (err-toward-false-negatives — no guessed links).
# --refresh-qobuz also needs the creds (to call album/get).
global QOBUZ_CREDS
if "qobuz" in cascade or args.refresh_qobuz:
QOBUZ_CREDS = resolve_qobuz(args, lambda m: print(m))
if not QOBUZ_CREDS:
cascade = [s for s in cascade if s != "qobuz"]
if args.refresh_qobuz:
sys.exit("[enrich] --refresh-qobuz needs Qobuz creds — aborting")
upgrade_from = {s.strip() for s in args.upgrade_from.split(",")
if s.strip() in SOURCES}
flacs = list(Path(args.library).rglob("*.flac"))
# --- one-shot migration: open.qobuz.com (streaming) -> purchase page -------
if args.refresh_qobuz:
refresh_qobuz_links(flacs, args, az_key)
return
print(f"[enrich] mode={'APPLY' if args.apply else 'DRY RUN'} cascade={cascade}"
+ (f" upgrade-from={sorted(upgrade_from)}" if upgrade_from else ""))
print(f"[enrich] {len(flacs)} FLAC files in library\n")
looked = found = upgraded = 0
by_source = {s: 0 for s in cascade}
touched = []
for p in flacs:
sp = str(p)
existing = flac_tag(sp, BUY_URL_TAG)
# Decide which sources to actually query for this file.
if existing and not args.force:
cur = url_source(existing)
if upgrade_from and cur in upgrade_from:
# Only try sources ranked ABOVE the current one — a same/lower
# match must never replace it. No-match leaves the link intact.
run_sources = cascade[:cascade.index(cur)] if cur in cascade else list(cascade)
if not run_sources:
continue
else:
continue # already tagged, not up for upgrade
else:
run_sources = cascade # untagged, or --force
artist = flac_tag(sp, "ARTIST").split(";")[0].strip() or flac_tag(sp, "ALBUMARTIST")
title = flac_tag(sp, "TITLE")
if not (artist and title):
continue
if args.limit and looked >= args.limit:
break
looked += 1
url = src = None
for s in run_sources:
url = SOURCES[s](artist, title)
time.sleep(sleep_for[s])
if url:
src = s
break
rel = sp[len(args.library):].lstrip("/")
if not url:
continue
found += 1
by_source[src] += 1
if existing:
upgraded += 1
print(f" ~ [{src}] {artist} - {title}\n -> {url}\n (replaces {existing})")
else:
print(f" + [{src}] {artist} - {title}\n -> {url}")
if args.apply:
if set_flac_tag(sp, BUY_URL_TAG, url):
touched.append(sp)
else:
print(" ! metaflac write failed")
else:
touched.append(sp)
if found % 50 == 0:
print(f" ...{found} links from {looked} lookups so far", flush=True)
print(f"\n[enrich] {looked} lookups, {found} {'tagged' if args.apply else 'would-tag'}"
f" ({', '.join(f'{s}={by_source[s]}' for s in cascade)})"
f" no-match={looked - found}"
+ (f" upgraded={upgraded}" if upgrade_from else ""))
if args.apply and touched and az_key:
print("[enrich] telling AzuraCast to reprocess touched files...")
az_reprocess(args.azuracast_base, az_key, args.station,
touched, args.library, lambda m: print(m))
elif args.apply and touched and not az_key:
print("[enrich] no AzuraCast key (flag/env/keyfile) — tags written; "
"AzuraCast's periodic media scan will pick them up.")
elif not args.apply:
print("[enrich] re-run with --apply to write (key via --azuracast-key, "
"$AZURACAST_API_KEY, or $ALEMBIC_CONFIG_DIR/pipeline/azuracast/api_key to reprocess)")
if __name__ == "__main__":
main()