Render playlist configs directly in-app; retire regen.sh and playlists.json (R3)

The app owns every input to a playlist's sldl .conf (playlists in its DB,
credentials in its encrypted store), so the old
DB -> playlists.json -> regen.sh subprocess -> regex credential patch-back
chain was three serialization hops for no reason. Collapse it:

- credential_service.render_playlist_confs(db) renders each <playlist>.conf
  from _template.conf in one pass: path placeholders substituted as literal
  text, then the four credential lines set, written 0600. This is now the
  single source of .conf rendering.
- playlist_service loses _write_playlists_json and regenerate_confs; _sync_to_disk
  just calls render_playlist_confs and syncs the scheduler. No subprocess, no
  intermediate JSON file.
- render_scope for spotify/soulseek re-renders confs via the same function
  (spotify still also writes _spotify.env for the python scripts). The dead
  _patch_conf_field / _every_playlist_conf helpers are removed.
- Delete pipeline/configs/regen.sh and drop it from the Dockerfile chmod;
  update _template.conf's comments.

Nothing outside playlist_service consumed regen.sh or playlists.json (verified).
Also closes the last remnants of the S2/S3 injection surface: names are
re-validated and path/credential values are substituted as literals, never
through sed or a shell.

Verified: end-to-end render in a throwaway DB (paths, creds incl. a password
with shell metacharacters, 0600, bad-name rejection) and a live re-render of
all 17 playlist configs with credentials preserved.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Dockerfile

@@ -47,7 +47,7 @@ COPY pipeline/ /app/pipeline/
 COPY schema/ /app/schema/
 COPY entrypoint.sh /app/entrypoint.sh
 
-RUN chmod +x /app/entrypoint.sh /app/pipeline/sldl /app/pipeline/bin/*.sh /app/pipeline/lib/*.sh /app/pipeline/lib/manual/* /app/pipeline/configs/regen.sh && \
+RUN chmod +x /app/entrypoint.sh /app/pipeline/sldl /app/pipeline/bin/*.sh /app/pipeline/lib/*.sh /app/pipeline/lib/manual/* && \
     chown -R alembic:alembic /app
 
 USER alembic

app/services/credential_service.py

@@ -95,25 +95,67 @@ def get_scope(db: Session, scope: str) -> dict[str, str]:
     return {row.key: crypto.decrypt(row.value_encrypted) for row in rows}
 
 
-def _patch_conf_field(path: Path, key: str, value: str) -> None:
-    """Rewrite a single `key = value` line in an sldl .conf file, leaving
-    every other line (including PLAYLIST_NAME/SPOTIFY_URL substitutions
-    regen.sh already applied) untouched."""
-    text = path.read_text()
-    # A newline in the value would inject an extra `key = value` line into the
-    # sldl config. Credentials never legitimately contain one, so strip any.
+# A playlist name is also its .conf filename; it is validated to this charset
+# at creation (playlist_service.validate_name). Re-checked here as defense in
+# depth so a bad name can never escape the config dir or inject config text.
+_SAFE_CONF_NAME = re.compile(r"^[A-Za-z0-9 _-]{1,64}$")
+
+# The credential lines the renderer fills in from the encrypted store. Everything
+# else in a rendered .conf comes verbatim from _template.conf.
+_CONF_CRED_KEYS = ("user", "pass", "spotify-id", "spotify-secret")
+
+
+def _set_conf_field(text: str, key: str, value: str) -> str:
+    """Return `text` with the single `key = ...` line set to `value`,
+    appending the line if absent. Newlines in the value are stripped so a
+    credential can't inject an extra config line."""
     value = value.replace("\r", "").replace("\n", "")
     pattern = re.compile(rf"^{re.escape(key)}\s*=.*$", re.MULTILINE)
     new_line = f"{key} = {value}"
     if pattern.search(text):
-        text = pattern.sub(new_line, text)
-    else:
-        text = text.rstrip("\n") + f"\n{new_line}\n"
-    path.write_text(text)
+        return pattern.sub(new_line, text)
+    return text.rstrip("\n") + f"\n{new_line}\n"
 
 
-def _every_playlist_conf() -> list[Path]:
-    return sorted(settings.pipeline_config_dir.glob("*.conf"))
+def render_playlist_confs(db: Session) -> None:
+    """Render every playlist's sldl .conf directly from _template.conf, with
+    the path placeholders and the Soulseek/Spotify credentials substituted in
+    one pass, written 0600.
+
+    This is the single source of .conf rendering. It replaces the older
+    DB -> playlists.json -> regen.sh -> regex-patch-back chain: the app owns
+    every input (playlists in its DB, credentials in its encrypted store), so
+    there is no reason to round-trip through an intermediate file and a
+    subprocess. Called on any playlist change (playlist_service) and on any
+    Spotify/Soulseek credential change (render_scope)."""
+    from app.services import playlist_service
+
+    template = (settings.pipeline_dir / "configs" / "_template.conf").read_text()
+    dropbox_root = str(settings.music_data_dir / "sldl-dropbox")
+    spotify = get_scope(db, "spotify")
+    soulseek = get_scope(db, "soulseek")
+    creds = {
+        "user": soulseek.get("username", ""),
+        "pass": soulseek.get("password", ""),
+        "spotify-id": spotify.get("client_id", ""),
+        "spotify-secret": spotify.get("client_secret", ""),
+    }
+    # Path placeholders are substituted as LITERAL text in a single left-to-right
+    # pass (never re-scanned), so a value can't be reinterpreted as another
+    # placeholder. Credential values are set as whole lines afterwards.
+    token_re = re.compile("|".join(map(re.escape, ("PLAYLIST_NAME", "SPOTIFY_URL", "SLDL_DROPBOX_ROOT"))))
+
+    settings.pipeline_config_dir.mkdir(parents=True, exist_ok=True)
+    for p in playlist_service.list_all(db):
+        if not _SAFE_CONF_NAME.match(p.name):
+            continue
+        mapping = {"PLAYLIST_NAME": p.name, "SPOTIFY_URL": p.spotify_url, "SLDL_DROPBOX_ROOT": dropbox_root}
+        text = token_re.sub(lambda m: mapping[m.group(0)], template)
+        for key in _CONF_CRED_KEYS:
+            text = _set_conf_field(text, key, creds[key])
+        out = settings.pipeline_config_dir / f"{p.name}.conf"
+        out.write_text(text)
+        out.chmod(0o600)
 
 
 def _write_env_file(path: Path, values: dict[str, str]) -> None:
@@ -135,20 +177,16 @@ def render_scope(db: Session, scope: str) -> None:
     if scope == "spotify":
         cid = values.get("client_id", "")
         csec = values.get("client_secret", "")
-        for conf in _every_playlist_conf():
-            _patch_conf_field(conf, "spotify-id", cid)
-            _patch_conf_field(conf, "spotify-secret", csec)
+        # _spotify.env is read by the pipeline python scripts (spotify-genre etc.).
         _write_env_file(
             settings.pipeline_config_dir / "_spotify.env",
             {"SPOTIFY_CLIENT_ID": cid, "SPOTIFY_CLIENT_SECRET": csec},
         )
+        # Re-render every playlist .conf so the new Spotify creds land in them.
+        render_playlist_confs(db)
 
     elif scope == "soulseek":
-        user = values.get("username", "")
-        pw = values.get("password", "")
-        for conf in _every_playlist_conf():
-            _patch_conf_field(conf, "user", user)
-            _patch_conf_field(conf, "pass", pw)
+        render_playlist_confs(db)
 
     elif scope == "navidrome":
         _write_env_file(

app/services/playlist_service.py

@@ -1,6 +1,5 @@
 import json
 import re
-import subprocess
 import time
 
 from sqlalchemy import select
@@ -127,7 +126,6 @@ def delete(db: Session, playlist_id: int) -> None:
     db.commit()
     conf_path = settings.pipeline_config_dir / f"{name}.conf"
     conf_path.unlink(missing_ok=True)
-    _write_playlists_json(db)
 
     from app.services import scheduler_service
 
@@ -168,65 +166,18 @@ def seed_legacy(db: Session) -> int:
     return created
 
 
-def _write_playlists_json(db: Session) -> None:
-    """The file regen.sh reads. Every active-or-not playlist is included
-    (see regen.sh's header comment) so a manual run is always possible;
-    the scheduler is what actually skips inactive ones."""
-    playlists = list_all(db)
-    payload = [
-        {
-            "name": p.name,
-            "spotify_url": p.spotify_url,
-            "active": p.active,
-            "no_m3u": p.no_m3u,
-        }
-        for p in playlists
-    ]
-    settings.pipeline_config_dir.mkdir(parents=True, exist_ok=True)
-    out_path = settings.pipeline_config_dir / "playlists.json"
-    out_path.write_text(json.dumps(payload, indent=2))
-
-
-def regenerate_confs(db: Session) -> subprocess.CompletedProcess:
-    """Write playlists.json, then run regen.sh to render/refresh every
-    playlist's .conf file. Newly-created .conf files still have unpatched
-    SOULSEEK_USER/SOULSEEK_PASS/SPOTIFY_CLIENT_ID/SPOTIFY_CLIENT_SECRET
-    placeholders at this point -- _sync_to_disk() (this function's caller)
-    re-renders credentials into them immediately afterward."""
-    _write_playlists_json(db)
-    regen_script = settings.pipeline_dir / "configs" / "regen.sh"
-    return subprocess.run(
-        [str(regen_script)],
-        capture_output=True,
-        text=True,
-        env={
-            "ALEMBIC_CONFIG_DIR": str(settings.alembic_config_dir),
-            "MUSIC_DATA_DIR": str(settings.music_data_dir),
-            "PIPELINE_DIR": str(settings.pipeline_dir),
-            # Must include /opt/venv/bin: regen.sh shells out to `python3`
-            # to parse playlists.json, and that's where this image's Python
-            # actually lives (confirmed against the real container -- a
-            # bare /usr/bin:/bin PATH made every regen.sh call silently
-            # generate zero .conf files, no error surfaced to the caller).
-            "PATH": "/opt/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-        },
-    )
-
-
 def _sync_to_disk(db: Session) -> None:
-    """Convenience wrapper used by create()/update(): regenerate confs,
-    immediately re-patch credentials into any newly-rendered file, and push
-    the change to the live scheduler if one is running -- this is what
-    makes add/remove-in-the-UI take effect without a redeploy."""
-    regenerate_confs(db)
-    # Imported lazily to avoid a circular import (credential_service and
-    # scheduler_service don't depend on playlist_service, but importing at
-    # module load time would still work here — done lazily anyway to keep
-    # the dependency direction obvious as all three services grow).
+    """Called by create()/update()/delete(): re-render every playlist's sldl
+    .conf (with credentials substituted, in-app -- no regen.sh subprocess or
+    playlists.json round-trip anymore) and push the change to the live
+    scheduler if one is running. This is what makes add/remove-in-the-UI take
+    effect without a redeploy.
+
+    Imported lazily to keep the dependency direction obvious (credential_service
+    and scheduler_service don't import playlist_service at module load)."""
     from app.services import credential_service, scheduler_service
 
-    credential_service.render_scope(db, "soulseek")
-    credential_service.render_scope(db, "spotify")
+    credential_service.render_playlist_confs(db)
 
     scheduler = scheduler_service.get_scheduler()
     if scheduler is not None:

pipeline/configs/_template.conf

@@ -4,15 +4,17 @@
 # sldl runs as a subprocess inside the alembic container (same container as
 # beets, on gluetun's network namespace for VPN-routed Soulseek P2P).
 
+# This is a TEMPLATE. The app renders one <playlist>.conf per playlist from it
+# (credential_service.render_playlist_confs), substituting PLAYLIST_NAME,
+# SPOTIFY_URL, SLDL_DROPBOX_ROOT and the credential lines below in one pass.
+# The placeholder values here are overwritten at render time; edit the
+# non-placeholder settings (quality, timeouts, ports) to change every playlist.
+
 # ==== Soulseek credentials ====
-# Substituted by the app's credential_service (not regen.sh) whenever the
-# soulseek secret is saved/updated — every rendered <playlist>.conf gets
-# re-patched at that point, independent of playlist add/remove.
 user = SOULSEEK_USER
 pass = SOULSEEK_PASS
 
 # ==== Spotify API credentials ====
-# Substituted by credential_service the same way, on the spotify secret.
 spotify-id = SPOTIFY_CLIENT_ID
 spotify-secret = SPOTIFY_CLIENT_SECRET
 
@@ -21,8 +23,8 @@ input = SPOTIFY_URL
 input-type = spotify
 
 # ==== Output paths ====
-# SLDL_DROPBOX_ROOT is substituted by regen.sh at render time from
-# $MUSIC_DATA_DIR (sldl's own config format has no env-var expansion).
+# SLDL_DROPBOX_ROOT is substituted at render time from $MUSIC_DATA_DIR
+# (sldl's own config format has no env-var expansion).
 path = SLDL_DROPBOX_ROOT/PLAYLIST_NAME
 playlist-path = SLDL_DROPBOX_ROOT/PLAYLIST_NAME/_sldl.m3u8
 write-playlist = true

pipeline/configs/regen.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-# regen.sh — regenerate per-playlist sldl configs from _template.conf.
-#
-# Source of truth is $ALEMBIC_CONFIG_DIR/pipeline/playlists.json, written by
-# the alembic app's playlist_service whenever a playlist is added/edited/
-# removed via the UI. This script never edits that source of truth — it only
-# renders it. Run automatically by the app before each playlist sync, or
-# manually: regen.sh
-#
-# playlists.json shape: a JSON array of objects, each with at least
-# {"name": "...", "spotify_url": "...", "active": true, "no_m3u": false}.
-# Inactive playlists still get a .conf rendered (so a manual run is always
-# possible) but are skipped by the scheduler.
-
-set -euo pipefail
-
-ALEMBIC_CONFIG_DIR="${ALEMBIC_CONFIG_DIR:-/config}"
-MUSIC_DATA_DIR="${MUSIC_DATA_DIR:-/data/music}"
-PIPELINE_DIR="${PIPELINE_DIR:-/app/pipeline}"
-
-CONFIG_OUT_DIR="$ALEMBIC_CONFIG_DIR/pipeline"
-PLAYLISTS_JSON="$CONFIG_OUT_DIR/playlists.json"
-TEMPLATE="$PIPELINE_DIR/configs/_template.conf"
-SLDL_DROPBOX_ROOT="$MUSIC_DATA_DIR/sldl-dropbox"
-
-if [[ ! -f "$PLAYLISTS_JSON" ]]; then
-  echo "ERROR: $PLAYLISTS_JSON not found — nothing to regenerate." >&2
-  echo "       This file is written by the alembic app; run its playlist" >&2
-  echo "       import/seed step first if this is a fresh setup." >&2
-  exit 1
-fi
-
-mkdir -p "$CONFIG_OUT_DIR"
-
-# Render each playlist's .conf entirely in Python. The playlist name and URL
-# are substituted as LITERAL text (single-pass regex replace, never re-scanned),
-# not fed to `sed` — an earlier sed-based version let a crafted playlist name
-# execute shell via GNU sed's `e` flag. Names are also re-validated here as
-# defense in depth even though the app validates them before writing the JSON.
-python3 - "$PLAYLISTS_JSON" "$TEMPLATE" "$CONFIG_OUT_DIR" "$SLDL_DROPBOX_ROOT" <<'PY'
-import json, os, re, sys
-
-playlists_json, template_path, out_dir, dropbox_root = sys.argv[1:5]
-
-with open(playlists_json) as f:
-    playlists = json.load(f)
-with open(template_path) as f:
-    template = f.read()
-
-SAFE_NAME = re.compile(r"^[A-Za-z0-9 _-]{1,64}$")
-
-for p in playlists:
-    name = p.get("name", "")
-    url = p.get("spotify_url", "")
-    if not SAFE_NAME.match(name):
-        print(f"WARNING: skipping playlist with unsafe name: {name!r}", file=sys.stderr)
-        continue
-    mapping = {
-        "PLAYLIST_NAME": name,
-        "SPOTIFY_URL": url,
-        "SLDL_DROPBOX_ROOT": dropbox_root,
-    }
-    pattern = re.compile("|".join(re.escape(k) for k in mapping))
-    rendered = pattern.sub(lambda m: mapping[m.group(0)], template)
-    out_path = os.path.join(out_dir, name + ".conf")
-    with open(out_path, "w") as f:
-        f.write(rendered)
-    # Holds the Soulseek password and Spotify secret once credentials are
-    # patched in, so keep it owner-only from the moment it is created.
-    os.chmod(out_path, 0o600)
-    print(f"Generated {out_path}")
-PY