Stage 0: migrate pipeline scripts from /opt/sldl, scaffold repo
Moves all ~30 pipeline scripts, configs, and the vendored sldl binary into this repo (source /opt/sldl left untouched). Removes all docker exec/docker compose dependencies now that beets and sldl run in-process/as a subprocess of this container instead of via soulbeet/on-demand sldl containers. Replaces hardcoded host paths, Navidrome credentials, and Spotify credential sourcing with env-var-driven paths and shared credential loaders. Adds Dockerfile, entrypoint.sh, requirements.txt, docker-compose.snippet.yml, and the initial app DB schema.
This commit is contained in:
andrew
Executable
+310
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
export-laptop-playlists.py — daily Navidrome → Traktor M3U export.
|
||||
|
||||
Lists every Navidrome playlist whose name starts with `dj-`, fetches each
|
||||
playlist's tracks via Subsonic, looks up the real on-disk path of each
|
||||
track in beets, and writes a `.m3u` for Syncthing → Traktor.
|
||||
|
||||
Output dir: $MUSIC_DATA_DIR/playlists-laptop/
|
||||
- One `dj-<name>.m3u` per Navidrome dj-* playlist (overwritten each run)
|
||||
- Exact format Traktor Pro 4 itself uses when you Export Playlist as M3U:
|
||||
one absolute Windows path per line (`C:\\Music\\Library\\...\\Track.flac`),
|
||||
UTF-8 encoded, LF line endings, NO `#EXTM3U` header. Traktor's M3U
|
||||
importer treats `#EXTM3U` as either a parse error or a phantom track
|
||||
path, which kicks the rest of the import into "treat-as-new" mode and
|
||||
the playlist ends up with unanalyzed records. With the header omitted,
|
||||
Traktor parses each line as a LOCATION, matches against the existing
|
||||
Collection by DIR+FILE+VOLUME, and the playlist inherits BPM, waveform,
|
||||
and key from the already-analyzed entries.
|
||||
|
||||
Why beets, not Subsonic, for paths: Navidrome's Subsonic `path` attribute
|
||||
is a *synthesized* string like `Artist/Album/01 - Title.flac` even when
|
||||
the real file is just `Artist/Album/Title.flac`. The track-number prefix
|
||||
is generated from metadata, not read from disk. Traktor then can't find
|
||||
the file. Beets stores the real path, so we look up each track by
|
||||
(album, title) in a one-shot beets index, falling back to artist
|
||||
disambiguation when multiple albums share the same album+title.
|
||||
|
||||
Reconciles deletes: any `dj-*.m3u` on disk whose stem is no longer a
|
||||
current Navidrome playlist is removed; also sweeps legacy `.m3u8`/`.nml`
|
||||
files from earlier experiments.
|
||||
|
||||
Existing M3U files in `$MUSIC_DATA_DIR/playlists/` (Navidrome's M3U
|
||||
import source for the automated Spotify pipeline) are NOT touched.
|
||||
|
||||
Usage:
|
||||
export-laptop-playlists.py [--verbose]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
_ALEMBIC_CONFIG_DIR = os.environ.get("ALEMBIC_CONFIG_DIR", "/config")
|
||||
_MUSIC_DATA_DIR = os.environ.get("MUSIC_DATA_DIR", "/data/music")
|
||||
|
||||
OUT_DIR = f"{_MUSIC_DATA_DIR}/playlists-laptop"
|
||||
|
||||
# Absolute Windows path to the laptop's library root.
|
||||
LIBRARY_LAPTOP_ROOT = r"C:\Music\Library"
|
||||
|
||||
# Beets stores paths with this prefix (the container-side mount of the
|
||||
# library); strip it to get the path relative to the library root. This is
|
||||
# still "/music/" through the migration's Stage 0-3 transitional mount;
|
||||
# revisit at Stage 4 once beets' directory: becomes MUSIC_DATA_DIR/Library.
|
||||
BEETS_LIBRARY_PREFIX = "/music/"
|
||||
|
||||
M3U_EXT = ".m3u"
|
||||
# Older formats from earlier iterations of this script — swept by reconcile.
|
||||
LEGACY_EXTS = (".m3u8", ".nml")
|
||||
|
||||
|
||||
def _load_navidrome_env():
|
||||
path = f"{_ALEMBIC_CONFIG_DIR}/pipeline/navidrome/admin.env"
|
||||
env = {}
|
||||
if os.path.exists(path):
|
||||
for line in open(path):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env[k] = v.strip().strip("'").strip('"')
|
||||
return env
|
||||
|
||||
_nd_env = _load_navidrome_env()
|
||||
ND_BASE = _nd_env.get("ND_BASE", "http://navidrome:4533")
|
||||
ND_USER = _nd_env.get("ND_USER", "andrew")
|
||||
ND_PASS = _nd_env.get("ND_PASS", "")
|
||||
ND_API_VERSION = "1.16.0"
|
||||
ND_CLIENT = "dj-export"
|
||||
|
||||
NS = "{http://subsonic.org/restapi}"
|
||||
|
||||
# ASCII Unit Separator — safe field separator for beets `-f` output.
|
||||
SEP = "\x1f"
|
||||
|
||||
|
||||
def _call(endpoint: str, params: dict[str, str]) -> ET.Element:
|
||||
qp = {
|
||||
"u": ND_USER, "p": ND_PASS,
|
||||
"v": ND_API_VERSION, "c": ND_CLIENT, "f": "xml",
|
||||
**params,
|
||||
}
|
||||
url = f"{ND_BASE}/rest/{endpoint}.view?{urllib.parse.urlencode(qp)}"
|
||||
with urllib.request.urlopen(url, timeout=60) as r:
|
||||
body = r.read()
|
||||
root = ET.fromstring(body)
|
||||
if root.attrib.get("status") != "ok":
|
||||
err = root.find(f"{NS}error")
|
||||
msg = err.attrib.get("message", "unknown") if err is not None else "?"
|
||||
raise RuntimeError(f"Subsonic {endpoint} failed: {msg}")
|
||||
return root
|
||||
|
||||
|
||||
def list_dj_playlists() -> list[tuple[str, str]]:
|
||||
root = _call("getPlaylists", {})
|
||||
out: list[tuple[str, str]] = []
|
||||
for p in root.iter(f"{NS}playlist"):
|
||||
name = p.attrib.get("name", "")
|
||||
if name.startswith("dj-"):
|
||||
out.append((p.attrib["id"], name))
|
||||
return out
|
||||
|
||||
|
||||
def get_playlist_tracks(pid: str) -> list[tuple[str, str, str]]:
|
||||
"""Return (albumartist, album, title) tuples per playlist entry."""
|
||||
root = _call("getPlaylist", {"id": pid})
|
||||
out: list[tuple[str, str, str]] = []
|
||||
for entry in root.iter(f"{NS}entry"):
|
||||
aa = (entry.attrib.get("displayAlbumArtist")
|
||||
or entry.attrib.get("artist", ""))
|
||||
album = entry.attrib.get("album", "")
|
||||
title = entry.attrib.get("title", "")
|
||||
out.append((aa, album, title))
|
||||
return out
|
||||
|
||||
|
||||
def _normkey(s: str) -> str:
|
||||
"""Lowercase + normalize curly quotes to straight ASCII."""
|
||||
return (s.lower()
|
||||
.replace("’", "'").replace("‘", "'")
|
||||
.replace("“", '"').replace("”", '"'))
|
||||
|
||||
|
||||
def build_beets_index() -> dict[tuple[str, str], list[tuple[str, str, str, str]]]:
|
||||
"""Map (album, title) →
|
||||
[(albumartist, album, title, library-relative path), ...].
|
||||
|
||||
Album and title are returned with original casing (not _normkey'd) so the
|
||||
NML COLLECTION entries can carry the real metadata for display."""
|
||||
fmt = SEP.join(["$albumartist", "$album", "$title", "$path"])
|
||||
res = subprocess.run(
|
||||
["beet", "ls", "-f", fmt],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
idx: dict[tuple[str, str], list[tuple[str, str, str, str]]] = {}
|
||||
for ln in res.stdout.splitlines():
|
||||
parts = ln.split(SEP)
|
||||
if len(parts) != 4:
|
||||
continue
|
||||
aa, alb, ti, path = parts
|
||||
if not path.startswith(BEETS_LIBRARY_PREFIX):
|
||||
continue
|
||||
rel = path[len(BEETS_LIBRARY_PREFIX):]
|
||||
idx.setdefault((_normkey(alb), _normkey(ti)), []).append((aa, alb, ti, rel))
|
||||
return idx
|
||||
|
||||
|
||||
def resolve(idx: dict[tuple[str, str], list[tuple[str, str, str, str]]],
|
||||
aa_query: str, album: str, title: str
|
||||
) -> tuple[str, str, str, str] | None:
|
||||
"""Return (albumartist, album, title, rel_path) for a Subsonic track, or None."""
|
||||
candidates = idx.get((_normkey(album), _normkey(title)))
|
||||
if not candidates:
|
||||
return None
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
q = _normkey(aa_query).replace(" • ", " ").replace(";", " ")
|
||||
for c in candidates:
|
||||
if _normkey(c[0]) in q:
|
||||
return c
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def to_windows_path(rel: str) -> str:
|
||||
"""Library-relative POSIX path → absolute Windows path with backslashes."""
|
||||
return LIBRARY_LAPTOP_ROOT + "\\" + rel.replace("/", "\\")
|
||||
|
||||
|
||||
def write_m3u(name: str, tracks: list[tuple[str, str, str, str]], verbose: bool) -> int:
|
||||
"""Plain-path M3U, no `#EXTM3U` header — matches the format Traktor
|
||||
Pro 4 itself emits via Export Playlist → M3U. With this byte-exact
|
||||
format, Traktor's import re-uses existing analyzed Collection entries
|
||||
(matched by DIR+FILE+VOLUME) instead of creating fresh unanalyzed
|
||||
records."""
|
||||
target = os.path.join(OUT_DIR, f"{name}{M3U_EXT}")
|
||||
tmp = target + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8", newline="\n") as f:
|
||||
for _aa, _alb, _ti, rel in tracks:
|
||||
f.write(to_windows_path(rel) + "\n")
|
||||
os.replace(tmp, target)
|
||||
if verbose:
|
||||
print(f" wrote {target} ({len(tracks)} tracks)")
|
||||
return len(tracks)
|
||||
|
||||
|
||||
def reconcile(current_names: set[str], verbose: bool) -> int:
|
||||
"""Delete dj-*.m3u files whose stem is no longer current, and sweep
|
||||
any legacy .m3u8/.nml leftovers from earlier iterations."""
|
||||
removed = 0
|
||||
if not os.path.isdir(OUT_DIR):
|
||||
return 0
|
||||
for entry in os.scandir(OUT_DIR):
|
||||
if not entry.is_file() or not entry.name.startswith("dj-"):
|
||||
continue
|
||||
if entry.name.endswith(M3U_EXT):
|
||||
stem = entry.name[:-len(M3U_EXT)]
|
||||
if stem in current_names:
|
||||
continue
|
||||
elif any(entry.name.endswith(e) for e in LEGACY_EXTS):
|
||||
pass # always sweep
|
||||
else:
|
||||
continue
|
||||
os.remove(entry.path)
|
||||
removed += 1
|
||||
if verbose:
|
||||
print(f" removed stale {entry.path}")
|
||||
return removed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--verbose", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
stamp = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
print(f"[{stamp}] export-laptop-playlists starting")
|
||||
|
||||
idx = build_beets_index()
|
||||
print(f" beets index: {len(idx)} (album,title) keys")
|
||||
|
||||
playlists = list_dj_playlists()
|
||||
print(f" found {len(playlists)} dj-* playlists in Navidrome")
|
||||
|
||||
# Safety guard: if Navidrome answers successfully but reports ZERO dj-*
|
||||
# playlists while exported files exist on disk, something is wrong on the
|
||||
# Navidrome side (fresh/restored DB, purge, renamed playlists). Running
|
||||
# reconcile() would delete every dj-*.m3u and Syncthing would propagate
|
||||
# the deletions to the laptop — the 2026-06-01 Traktor wipe through the
|
||||
# other door. Refuse and keep the files.
|
||||
if not playlists:
|
||||
existing = [e.name for e in os.scandir(OUT_DIR)
|
||||
if e.is_file() and e.name.startswith("dj-")]
|
||||
if existing:
|
||||
print(f" WARN: Navidrome returned 0 dj-* playlists but "
|
||||
f"{len(existing)} dj-* file(s) exist in {OUT_DIR} — "
|
||||
f"REFUSING to reconcile/delete. Check Navidrome before "
|
||||
f"trusting this.")
|
||||
return 0
|
||||
|
||||
total_tracks = 0
|
||||
total_unresolved = 0
|
||||
written = 0
|
||||
skipped = 0
|
||||
for pid, name in playlists:
|
||||
try:
|
||||
tracks = get_playlist_tracks(pid)
|
||||
except Exception as e:
|
||||
print(f" WARN: failed to fetch {name}: {e}")
|
||||
continue
|
||||
resolved: list[tuple[str, str, str, str]] = []
|
||||
unresolved: list[tuple[str, str, str]] = []
|
||||
for aa, album, title in tracks:
|
||||
hit = resolve(idx, aa, album, title)
|
||||
if hit is None:
|
||||
unresolved.append((aa, album, title))
|
||||
continue
|
||||
resolved.append(hit)
|
||||
if unresolved:
|
||||
total_unresolved += len(unresolved)
|
||||
print(f" WARN: {name}: {len(unresolved)} track(s) not in beets index "
|
||||
f"(skipped). First: {unresolved[0]}")
|
||||
# Safety guard: never blow away a populated .m3u with an empty export.
|
||||
# On 2026-06-01 a PURGEMISSING scan emptied the Navidrome dj-* playlists
|
||||
# while the QNAP share was flaky; this script then overwrote every
|
||||
# laptop .m3u with 0 tracks, and the laptop's auto-merge wiped Traktor.
|
||||
# If Navidrome resolves to 0 tracks but a non-empty file already exists,
|
||||
# keep the file and warn loudly instead of trusting the empty result.
|
||||
if not resolved:
|
||||
target = os.path.join(OUT_DIR, f"{name}{M3U_EXT}")
|
||||
if os.path.exists(target) and os.path.getsize(target) > 0:
|
||||
print(f" WARN: {name}: Navidrome returned 0 resolvable tracks but "
|
||||
f"{target} is non-empty — REFUSING to overwrite (kept existing). "
|
||||
f"Check Navidrome (likely a purge/scan glitch) before trusting this.")
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
total_tracks += write_m3u(name, resolved, args.verbose)
|
||||
written += 1
|
||||
|
||||
current_names = {name for _pid, name in playlists}
|
||||
removed = reconcile(current_names, args.verbose)
|
||||
|
||||
print(f"[{stamp}] done. wrote {written} playlists, {total_tracks} total tracks, "
|
||||
f"{total_unresolved} unresolved, {skipped} kept (empty-guard), "
|
||||
f"removed {removed} stale files")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user