From e5d9c7f0434f1391991e6f3cacd6f442e4a0bdd7 Mon Sep 17 00:00:00 2001 From: andrew Date: Tue, 14 Jul 2026 11:16:55 -0600 Subject: [PATCH] 0.6: Spotify OAuth connect flow, AzuraCast base URL, fingerprint/buy-url fixes - Add "Connect Spotify" OAuth flow (/connect/spotify) so newly created Spotify apps can read playlists: Spotify now requires a user token for playlist reads, which client-credentials alone can no longer provide. The stored refresh token is rendered as spotify-refresh into each playlist's sldl .conf -- sldl's own docs confirm supplying it skips its interactive login flow, which is what was causing multi-hour hangs on a stuck sync. Grandfathered apps keep working unchanged if no account is connected. - Fix the connect flow's redirect_uri: request.url_for() reflected the raw connection scheme (http) rather than what the reverse proxy actually served over, which Spotify's exact-match redirect_uri check rejected. Force https rather than trust the unproxied scheme. - Add an AzuraCast base URL credential field alongside the API key, with a keyfile fallback so the scheduled enrich-buy-url.py job can actually reach AzuraCast (previously it had no way to receive a base URL at all when run from the scheduler, so the reprocess call was silently always skipped). - Fix build-fingerprint-index.py exiting non-zero on any single fingerprint failure instead of only when nothing was written. - Guard enrich-buy-url.py's AzuraCast reprocess against an empty --azuracast-base (raised ValueError since PD2 removed the hardcoded base). Co-Authored-By: Claude Sonnet 5 --- .gitignore | 1 + README.md | 25 +++-- app/main.py | 3 +- app/routers/connect.py | 100 +++++++++++++++++++ app/services/credential_service.py | 32 ++++-- app/services/spotify_client.py | 31 ++++-- app/templates/credentials/index.html | 15 +++ app/templates/dashboard.html | 2 +- pipeline/bin/run-playlist.sh | 8 +- pipeline/configs/_template.conf | 6 ++ pipeline/lib/_spotify_auth.py | 20 ++-- pipeline/lib/build-fingerprint-index.py | 2 +- pipeline/lib/enrich-buy-url.py | 21 +++- pipeline/lib/fix-track-metadata.py | 3 +- pipeline/lib/manual/backfill-spotify-tags.sh | 20 +++- pipeline/lib/spotify-genre.py | 6 +- pipeline/lib/spotify-retag.py | 5 +- tests/test_credential_service.py | 27 +++++ 18 files changed, 280 insertions(+), 47 deletions(-) create mode 100644 app/routers/connect.py diff --git a/.gitignore b/.gitignore index 77b66ea..4e2446d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ __pycache__/ *.pyc .pytest_cache/ +.venv/ *.db *.db-journal *.db-wal diff --git a/README.md b/README.md index a86f038..61d64c5 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Before you begin, make sure you have: 1. **A server that runs Docker**, with `docker compose` available. Any Linux box, NAS, or mini PC works. 2. **A music server that reads a plain folder of tagged files**, such as [Navidrome](https://www.navidrome.org/). alembic builds your library on disk; Navidrome (or similar) is what you actually listen with. 3. **A Soulseek account.** This is how tracks get downloaded. Sign up free at [soulseek.com](https://www.soulseek.com/). -4. **Spotify credentials that can read playlists.** alembic reads your playlists' track lists from Spotify (it does not download from Spotify itself, only Soulseek). Important: because of Spotify's 2024/2025 API changes, a **brand-new** Spotify Developer app can no longer do this kind of app-only playlist read, so creating your own fresh app will not work. You need the Client ID and Secret from an app created before those changes, which in practice means borrowing them from someone already running alembic. They are read-only and expose no account access (see the credentials section). If nobody you know has a working app, alembic cannot read playlists today. +4. **A Spotify Developer app.** alembic reads your playlists' track lists from Spotify (it does not download from Spotify itself, only Soulseek). Create one free at [developer.spotify.com/dashboard](https://developer.spotify.com/dashboard) for the Client ID and Secret. Because of a 2025/2026 Spotify API change, a brand-new app can no longer read playlists with just those two values — you also need to **connect your Spotify account** (one click in Settings → Credentials, standard Spotify login) the first time. This is a one-time step; see the credentials section below. 5. **A login provider that speaks OpenID Connect (OIDC).** alembic doesn't have its own username/password login, it delegates to something you already trust. [Pocket ID](https://github.com/pocket-id/pocket-id) is a small self-hosted option built exactly for this, but Authentik, Keycloak, Authelia, or any other OIDC provider works too. Optional, add these later if you want them: @@ -73,10 +73,10 @@ Optional, add these later if you want them: Pull the prebuilt image onto your Docker host: ```bash -docker pull git.kretzer.club/andrew/alembic:0.5.2 +docker pull git.kretzer.club/andrew/alembic:0.6 ``` -That is the whole install. You do not need to download the source or build anything. The `0.5.2` is the version; you can pin to it so nothing changes under you, or use `latest` to always get the newest. +That is the whole install. You do not need to download the source or build anything. The `0.6` is the version; you can pin to it so nothing changes under you, or use `latest` to always get the newest. (If you would rather build it yourself from source, you can, but you do not need to.) @@ -136,7 +136,7 @@ Create a file called `docker-compose.yml` on your server (put it wherever you ke ```yaml services: alembic: - image: git.kretzer.club/andrew/alembic:0.5.2 + image: git.kretzer.club/andrew/alembic:0.6 container_name: alembic ports: - "8420:8420" @@ -191,14 +191,18 @@ Spotify, Soulseek, and Navidrome are marked **Required** — every playlist sync | Service | What to enter | Where to get it | |---|---|---| -| **Spotify** *(required)* | Client ID, Client Secret | Reads your playlists' track lists. A brand-new app created at [developer.spotify.com/dashboard](https://developer.spotify.com/dashboard) can no longer do this (see "What you need before you start" above and Troubleshooting below), so use the Client ID and Secret from an older app that still works. | +| **Spotify** *(required)* | Client ID, Client Secret, then **Connect Spotify** | Reads your playlists' track lists. Create an app at [developer.spotify.com/dashboard](https://developer.spotify.com/dashboard) for the Client ID/Secret, save them, then click **Connect Spotify** and log in with your Spotify account — required for any app created after Spotify's 2025/2026 API change (see Troubleshooting below). | | **Soulseek** *(required)* | Username, Password | Your normal Soulseek login. | | **Navidrome** *(required)* | Base URL, admin username, admin password | The address of your Navidrome server and an admin account on it. Used so alembic can trigger a library rescan after downloads. | | **Bandcamp** *(optional)* | Username, format preference, cookies | Export your Bandcamp cookies while logged into bandcamp.com in your browser, using an extension like "Get cookies.txt LOCALLY", and paste the contents in. Format preference is a space-separated list like `flac mp3-320`. | | **Qobuz** *(optional)* | Token, App ID, region | A logged-in Qobuz session token — used (alongside Bandcamp) as a source for "buy this" links stamped onto tracks you didn't purchase there. | -| **AzuraCast** *(optional)* | API key | Only relevant if you also run an AzuraCast radio station off the same library. It's not a buy-link source itself (that's Bandcamp/Qobuz, written straight to the file); this just lets alembic tell AzuraCast to immediately pick up a tag change instead of waiting for its own periodic scan. | +| **AzuraCast** *(optional)* | Base URL, API key | Only relevant if you also run an AzuraCast radio station off the same library. It's not a buy-link source itself (that's Bandcamp/Qobuz, written straight to the file); this just lets alembic tell AzuraCast to immediately pick up a tag change instead of waiting for its own periodic scan. Base URL is your AzuraCast instance's address, e.g. `https://radio.example.com`. | | **Telegram** *(optional)* | Bot token, chat ID | Create a bot via [@BotFather](https://t.me/BotFather) if you want the daily status digest sent to a chat. | +### Connecting your Spotify account + +After saving your Spotify Client ID and Secret, add this redirect URI in your Spotify app's dashboard settings (Developer Dashboard → your app → Settings → Redirect URIs): `https:///connect/spotify/callback`. Then click **Connect Spotify** on the Credentials page and log in — this only needs to be done once. It's what lets alembic read playlists on an app created after Spotify's 2025/2026 restriction on app-only playlist access; grandfathered apps continue to work without it, but connecting is harmless either way. + ## Adding your first playlist 1. Go to **Playlists** → **Add playlist**. @@ -220,7 +224,7 @@ That's it. On its next scheduled run (or immediately, using the **Run now** butt ## Updating alembic -When a new version is released, change the version in your `docker-compose.yml` (for example `:0.5.2` to the new number), then pull and restart: +When a new version is released, change the version in your `docker-compose.yml` (for example `:0.6` to the new number), then pull and restart: ```bash docker compose pull @@ -269,11 +273,10 @@ Double check the redirect URI registered with your OIDC provider exactly matches **A playlist says it's synced but nothing downloaded.** Check that playlist's job history under **Settings → Jobs**, the log will usually show whether Soulseek couldn't find a track, or a credential is missing. The playlist's own page will also show everything as "Waiting to download" if nothing came through. -**The playlist page shows "Spotify denied access (403)", or nothing downloads and the logs mention 403.** -Your Spotify app can't read the playlist. Two things to check: +**The playlist page shows "Spotify denied access (403)" or a 401, or nothing downloads and the logs mention one of those.** +Your Spotify app can't read the playlist with app-only (client-credentials) access alone. Spotify closed that off for apps created after its 2025/2026 API change, on both the old `/tracks` endpoint (403) and the current `/items` endpoint (401 "valid user authentication required") — this affects even public playlists. The "extended quota mode" that would lift it is granted only to organizations, not individuals, since May 2025. -- **The playlist must be public.** alembic reads playlists with app-only (client-credentials) access, which can only see public playlists, never private ones. Set the playlist to public on Spotify. -- **Your Spotify credentials must come from an app that can still read playlists app-only.** A brand-new Spotify Developer app cannot. Spotify has closed app-only playlist reads for new apps on both the old `/tracks` endpoint (returns 403) and the new `/items` endpoint (returns a 401 "valid user authentication required"). The "extended quota mode" that would lift this is granted only to organizations, not individuals, since May 2025. So a fresh personal app simply will not work here. Use the Client ID and Secret from an app created before these changes, for example from a friend already running alembic. Those credentials are read-only, expose no account access, and only share the app's rate limit. +The fix: go to **Settings → Credentials** and click **Connect Spotify** (see "Connecting your Spotify account" above). This mints a user token that works regardless of when your app was created. If you haven't already, add the redirect URI `https:///connect/spotify/callback` in your Spotify app's dashboard first, or the connect step itself will fail. **Dedup found something that isn't actually a duplicate.** Use the "Keep both" button on that pair. alembic will remember your decision and won't flag that exact pair again. diff --git a/app/main.py b/app/main.py index f6e58e3..c29524b 100644 --- a/app/main.py +++ b/app/main.py @@ -9,7 +9,7 @@ from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.middleware.sessions import SessionMiddleware from app.db import enable_beets_db_wal, init_db, mark_interrupted_runs -from app.routers import artist_casing, auth, credentials, dashboard, dedup, genres, health, import_, jobs, library, playlists +from app.routers import artist_casing, auth, connect, credentials, dashboard, dedup, genres, health, import_, jobs, library, playlists from app.security.session import resolve_session_secret from app.services import scheduler_service from app.settings import settings @@ -82,6 +82,7 @@ def create_app() -> FastAPI: app.include_router(genres.router) app.include_router(playlists.router) app.include_router(credentials.router) + app.include_router(connect.router) app.include_router(jobs.router) app.include_router(artist_casing.router) diff --git a/app/routers/connect.py b/app/routers/connect.py new file mode 100644 index 0000000..a4b36d5 --- /dev/null +++ b/app/routers/connect.py @@ -0,0 +1,100 @@ +"""Spotify OAuth "Connect Spotify" flow. + +Not on the /auth prefix or the /settings/credentials prefix: Spotify's app +dashboard needs one fixed redirect URI to register, and this is it in full: +https:///connect/spotify/callback. + +Unlike Pocket ID (app/security/oidc.py), Spotify's client id/secret live in +the encrypted credential store, not app.settings -- so this builds the +authorize/token requests directly with httpx instead of going through an +authlib client registered once at import time with fixed credentials. +""" +import base64 +import secrets +import urllib.parse + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse + +from app.db import get_db +from app.security.deps import require_auth +from app.services import credential_service + +router = APIRouter(tags=["spotify-connect"]) + +AUTHORIZE_URL = "https://accounts.spotify.com/authorize" +TOKEN_URL = "https://accounts.spotify.com/api/token" +SCOPES = "playlist-read-private playlist-read-collaborative" + + +def _callback_redirect_uri(request: Request) -> str: + """request.url_for() reflects the scheme uvicorn saw on the actual TCP + connection, not what the reverse proxy served over -- alembic sits behind + a TLS-terminating proxy (per every deployment example in the README) and + uvicorn isn't told to trust its X-Forwarded-Proto, so this comes back + `http://`. Spotify requires an exact scheme match against the redirect + URI registered in its app dashboard (always https for a real hostname), + so force it here rather than trust the raw connection's scheme.""" + return str(request.url_for("spotify_callback").replace(scheme="https")) + + +@router.get("/connect/spotify") +async def connect_spotify(request: Request, user: dict = Depends(require_auth), db=Depends(get_db)): + client_id = credential_service.get_scope(db, "spotify").get("client_id") + if not client_id: + # Nothing to connect to yet -- send them to save client_id/secret first. + return RedirectResponse(url="/settings/credentials", status_code=303) + + # CSRF guard on the callback: the state we get back must match what we + # handed out for this session, not just be present. + state = secrets.token_urlsafe(24) + request.session["spotify_oauth_state"] = state + + params = { + "client_id": client_id, + "response_type": "code", + "redirect_uri": _callback_redirect_uri(request), + "scope": SCOPES, + "state": state, + } + return RedirectResponse(url=f"{AUTHORIZE_URL}?{urllib.parse.urlencode(params)}") + + +@router.get("/connect/spotify/callback", name="spotify_callback") +async def spotify_callback(request: Request, user: dict = Depends(require_auth), db=Depends(get_db)): + if error := request.query_params.get("error"): + raise HTTPException(400, f"Spotify authorization failed: {error}") + + state = request.query_params.get("state") + if not state or state != request.session.pop("spotify_oauth_state", None): + raise HTTPException(400, "Spotify authorization state mismatch") + + code = request.query_params.get("code") + if not code: + raise HTTPException(400, "Spotify did not return an authorization code") + + creds = credential_service.get_scope(db, "spotify") + client_id = creds.get("client_id") + client_secret = creds.get("client_secret") + if not client_id or not client_secret: + raise HTTPException(400, "Spotify client id/secret not configured") + + basic = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode() + resp = httpx.post( + TOKEN_URL, + data={ + "grant_type": "authorization_code", + "code": code, + "redirect_uri": _callback_redirect_uri(request), + }, + headers={"Authorization": f"Basic {basic}"}, + timeout=15, + ) + resp.raise_for_status() + refresh_token = resp.json().get("refresh_token") + if not refresh_token: + raise HTTPException(400, "Spotify did not return a refresh token") + + credential_service.set_credentials(db, "spotify", {"refresh_token": refresh_token}) + return RedirectResponse(url="/settings/credentials", status_code=303) diff --git a/app/services/credential_service.py b/app/services/credential_service.py index 623b29d..2f14227 100644 --- a/app/services/credential_service.py +++ b/app/services/credential_service.py @@ -13,11 +13,11 @@ from app.settings import settings # Which fields exist per scope, and whether each is safe to display back to # the UI once saved (short opaque strings need never be shown again). SCOPE_FIELDS = { - "spotify": ["client_id", "client_secret"], + "spotify": ["client_id", "client_secret", "refresh_token"], "soulseek": ["username", "password"], "navidrome": ["base_url", "admin_user", "admin_pass"], "bandcamp": ["username", "format_pref", "cookies_txt"], - "azuracast": ["api_key"], + "azuracast": ["base_url", "api_key"], "qobuz": ["token", "app_id", "region"], "telegram": ["bot_token", "chat_id"], } @@ -56,6 +56,7 @@ SECRET_KEYS = { "api_key", "token", "bot_token", + "refresh_token", } @@ -116,7 +117,7 @@ _SAFE_CONF_NAME = re.compile(r"^[A-Za-z0-9 _-]{1,64}$") # The credential lines the renderer fills in from the encrypted store. Everything # else in a rendered .conf comes verbatim from _template.conf. -_CONF_CRED_KEYS = ("user", "pass", "spotify-id", "spotify-secret") +_CONF_CRED_KEYS = ("user", "pass", "spotify-id", "spotify-secret", "spotify-refresh") def _set_conf_field(text: str, key: str, value: str) -> str: @@ -153,6 +154,7 @@ def render_playlist_confs(db: Session) -> None: "pass": soulseek.get("password", ""), "spotify-id": spotify.get("client_id", ""), "spotify-secret": spotify.get("client_secret", ""), + "spotify-refresh": spotify.get("refresh_token", ""), } # Path placeholders are substituted as LITERAL text in a single left-to-right # pass (never re-scanned), so a value can't be reinterpreted as another @@ -191,10 +193,20 @@ def render_scope(db: Session, scope: str) -> None: if scope == "spotify": cid = values.get("client_id", "") csec = values.get("client_secret", "") - # _spotify.env is read by the pipeline python scripts (spotify-genre etc.). + # _spotify.env is read by the pipeline python scripts (spotify-genre, + # spotify-retag, fix-track-metadata). SPOTIFY_REFRESH_TOKEN is only set + # once an account is connected via /connect/spotify -- its presence is + # what switches every reader from client-credentials to a user token + # (see _spotify_auth.get_token). The rendered playlist .conf gets the + # same refresh token too (spotify-refresh, in render_playlist_confs), + # which is what lets sldl itself use a user token. _write_env_file( settings.pipeline_config_dir / "_spotify.env", - {"SPOTIFY_CLIENT_ID": cid, "SPOTIFY_CLIENT_SECRET": csec}, + { + "SPOTIFY_CLIENT_ID": cid, + "SPOTIFY_CLIENT_SECRET": csec, + "SPOTIFY_REFRESH_TOKEN": values.get("refresh_token", ""), + }, ) # Re-render every playlist .conf so the new Spotify creds land in them. render_playlist_confs(db) @@ -239,6 +251,10 @@ def render_scope(db: Session, scope: str) -> None: key_path = az_dir / "api_key" key_path.write_text(values.get("api_key", "")) key_path.chmod(0o600) + # base_url isn't secret, but still keyfile-based (not env) so it's + # readable via the same fallback cascade as api_key -- see + # enrich-buy-url.py's --azuracast-base resolution. + (az_dir / "base_url").write_text(values.get("base_url", "")) elif scope == "qobuz": qobuz_dir = settings.pipeline_config_dir / "qobuz" @@ -289,7 +305,9 @@ def _clear_rendered(scope: str) -> None: notify-telegram.sh call). Re-enabling calls render_scope() again to recreate the file from the still-stored values.""" if scope == "azuracast": - (settings.pipeline_config_dir / "azuracast" / "api_key").unlink(missing_ok=True) + az_dir = settings.pipeline_config_dir / "azuracast" + (az_dir / "api_key").unlink(missing_ok=True) + (az_dir / "base_url").unlink(missing_ok=True) elif scope == "qobuz": qobuz_dir = settings.pipeline_config_dir / "qobuz" for name in ("token", "app_id", "region"): @@ -360,6 +378,8 @@ def auth_states(db: Session) -> list[dict]: values = get_scope(db, scope) configured = bool(values) entry = {"scope": scope, "configured": configured} + if scope == "spotify": + entry["spotify_connected"] = bool(values.get("refresh_token")) if scope == "bandcamp" and values.get("cookies_txt"): expiry = _bandcamp_cookie_expiry(values["cookies_txt"]) if expiry is not None: diff --git a/app/services/spotify_client.py b/app/services/spotify_client.py index d33934e..04c0c36 100644 --- a/app/services/spotify_client.py +++ b/app/services/spotify_client.py @@ -11,8 +11,11 @@ TOKEN_URL = "https://accounts.spotify.com/api/token" API_BASE = "https://api.spotify.com/v1" # Module-level cache: one alembic process, one Spotify app registration -- -# a single shared client-credentials token is fine (no per-user tokens here). -_token_cache: dict = {"token": None, "expires_at": 0.0} +# a single shared token is fine (single-user app, no per-request identity). +# Keyed by mode too, so connecting/disconnecting an account (switching +# between a user token and a client-credentials token) can't serve a stale +# token minted under the other grant type. +_token_cache: dict = {"token": None, "expires_at": 0.0, "mode": None} _PLAYLIST_ID_RE = re.compile(r"playlist/([A-Za-z0-9]+)") @@ -26,19 +29,34 @@ def _extract_playlist_id(playlist_url: str) -> str: def _get_token(db: Session) -> str: now = time.time() - if _token_cache["token"] and _token_cache["expires_at"] > now + 30: - return _token_cache["token"] - creds = credential_service.get_scope(db, "spotify") client_id = creds.get("client_id") client_secret = creds.get("client_secret") if not client_id or not client_secret: raise RuntimeError("Spotify credentials not configured (settings/credentials)") + refresh_token = creds.get("refresh_token") + # A user token (from /connect/spotify) can read playlists on newly created + # Spotify apps; client-credentials can't (Spotify blocks GET /items for + # new apps without user auth). Prefer the user token whenever one is + # connected, falling back to client-credentials for grandfathered apps. + mode = "user" if refresh_token else "client_credentials" + + if ( + _token_cache["token"] + and _token_cache["mode"] == mode + and _token_cache["expires_at"] > now + 30 + ): + return _token_cache["token"] basic = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode() + grant_data = ( + {"grant_type": "refresh_token", "refresh_token": refresh_token} + if refresh_token + else {"grant_type": "client_credentials"} + ) resp = httpx.post( TOKEN_URL, - data={"grant_type": "client_credentials"}, + data=grant_data, headers={"Authorization": f"Basic {basic}"}, timeout=15, ) @@ -46,6 +64,7 @@ def _get_token(db: Session) -> str: data = resp.json() _token_cache["token"] = data["access_token"] _token_cache["expires_at"] = now + data["expires_in"] + _token_cache["mode"] = mode return _token_cache["token"] diff --git a/app/templates/credentials/index.html b/app/templates/credentials/index.html index db09ae2..0bb5595 100644 --- a/app/templates/credentials/index.html +++ b/app/templates/credentials/index.html @@ -45,8 +45,22 @@

Disabled — its scheduled automation is paused (or, for buy-link sources, its rendered credential file is removed) until you turn this back on. Saved values below are kept.

{% endif %} + {% if scope == "spotify" %} +
+ +

Newly created Spotify apps can no longer read playlists without this — connecting an account mints a user token that works where the app-only credentials above are blocked. Grandfathered apps keep working without it.

+ {% if "refresh_token" in configured.get(scope, []) %} + Connected + Reconnect Spotify + {% else %} + Connect Spotify + {% endif %} +
+ {% endif %} +
{% for field in fields %} + {% if not (scope == "spotify" and field == "refresh_token") %}
+ {% endif %} {% endfor %}
diff --git a/app/templates/dashboard.html b/app/templates/dashboard.html index 154ed85..6fe91a8 100644 --- a/app/templates/dashboard.html +++ b/app/templates/dashboard.html @@ -63,7 +63,7 @@

Credential status

    {% for a in auth_states %} -
  • {{ a.scope }}{% if a.days_left is defined %} {{ a.days_left }}d{% endif %}
    • +
  • {{ a.scope }}{% if a.days_left is defined %} {{ a.days_left }}d{% endif %}{% if a.spotify_connected %} (OAuth){% endif %}
    • {% endfor %}

Manage credentials →

diff --git a/pipeline/bin/run-playlist.sh b/pipeline/bin/run-playlist.sh index 6e05738..3342104 100755 --- a/pipeline/bin/run-playlist.sh +++ b/pipeline/bin/run-playlist.sh @@ -65,7 +65,11 @@ log "Running sldl for: $PLAYLIST_NAME" # which silently skips every later-scheduled playlist for the night. (Seen # 2026-06-18: a transient Spotify client-creds failure made sldl fall back to # interactive OAuth — "manually open: https://accounts.spotify.com/..." — and -# it waited 7h on a browser callback that never comes.) timeout TERMs the run +# it waited 7h on a browser callback that never comes. Fixed by always +# rendering `spotify-refresh` into the conf once a Spotify account is +# connected via /connect/spotify — sldl's own docs confirm supplying a +# refresh token skips the login-flow requirement entirely. This timeout stays +# as a backstop for any other cause of a stuck run.) timeout TERMs the run # and KILLs after a grace period; there's no leftover container to clean up # now that sldl is a plain subprocess instead of a docker-compose service. SLDL_TIMEOUT="${SLDL_TIMEOUT:-2700}" # 45 min; override via env @@ -115,10 +119,12 @@ find "$DROPBOX" -name "*.incomplete" -type f -delete 2>>"$LOG" || true SPOTIFY_URL=$(sed -n 's/^input *= *//p' "$CONFIG_FILE" | tr -d ' ') SPOTIFY_CLIENT_ID=$(sed -n 's/^spotify-id *= *//p' "$CONFIG_FILE" | tr -d ' ') SPOTIFY_CLIENT_SECRET=$(sed -n 's/^spotify-secret *= *//p' "$CONFIG_FILE" | tr -d ' ') +SPOTIFY_REFRESH_TOKEN=$(sed -n 's/^spotify-refresh *= *//p' "$CONFIG_FILE" | tr -d ' ') if [[ -n "$SPOTIFY_URL" && -n "$SPOTIFY_CLIENT_ID" && -n "$SPOTIFY_CLIENT_SECRET" ]]; then log "Rewriting tags from Spotify for files in $DROPBOX" if SPOTIFY_CLIENT_ID="$SPOTIFY_CLIENT_ID" SPOTIFY_CLIENT_SECRET="$SPOTIFY_CLIENT_SECRET" \ + SPOTIFY_REFRESH_TOKEN="$SPOTIFY_REFRESH_TOKEN" \ python3 ${PIPELINE_DIR:-/app/pipeline}/lib/spotify-retag.py "$SPOTIFY_URL" "$DROPBOX" >> "$LOG" 2>&1; then log "Spotify retag complete" else diff --git a/pipeline/configs/_template.conf b/pipeline/configs/_template.conf index 4264cea..fc3bf52 100644 --- a/pipeline/configs/_template.conf +++ b/pipeline/configs/_template.conf @@ -17,6 +17,12 @@ pass = SOULSEEK_PASS # ==== Spotify API credentials ==== spotify-id = SPOTIFY_CLIENT_ID spotify-secret = SPOTIFY_CLIENT_SECRET +# Set once an account is connected via /connect/spotify (blank otherwise). +# sldl's own docs confirm supplying a refresh token skips its interactive +# login-flow requirement entirely -- this is what lets newly created Spotify +# apps (which Spotify blocks from reading playlists with client-credentials +# alone) work without sldl ever trying to open a browser. +spotify-refresh = SPOTIFY_REFRESH_TOKEN # ==== Input (Spotify playlist URL) ==== input = SPOTIFY_URL diff --git a/pipeline/lib/_spotify_auth.py b/pipeline/lib/_spotify_auth.py index d569b41..b7ad30c 100644 --- a/pipeline/lib/_spotify_auth.py +++ b/pipeline/lib/_spotify_auth.py @@ -12,19 +12,27 @@ import urllib.parse import urllib.request _TOKEN_URL = "https://accounts.spotify.com/api/token" -_cache: dict[tuple[str, str], str] = {} +_cache: dict[tuple[str, str, str | None], str] = {} -def get_token(client_id: str, client_secret: str) -> str: - """Return a client-credentials access token, cached per (id, secret) for - the life of the process.""" - key = (client_id, client_secret) +def get_token(client_id: str, client_secret: str, refresh_token: str | None = None) -> str: + """Return an access token, cached per (id, secret, refresh_token) for the + life of the process. If refresh_token is set (an account has been + connected via /connect/spotify), mints a user token -- required for + playlist reads on newly created Spotify apps, which reject + client-credentials tokens. Otherwise falls back to client-credentials.""" + key = (client_id, client_secret, refresh_token) if key in _cache: return _cache[key] creds = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode() + grant = ( + {"grant_type": "refresh_token", "refresh_token": refresh_token} + if refresh_token + else {"grant_type": "client_credentials"} + ) req = urllib.request.Request( _TOKEN_URL, - data=urllib.parse.urlencode({"grant_type": "client_credentials"}).encode(), + data=urllib.parse.urlencode(grant).encode(), headers={ "Authorization": f"Basic {creds}", "Content-Type": "application/x-www-form-urlencoded", diff --git a/pipeline/lib/build-fingerprint-index.py b/pipeline/lib/build-fingerprint-index.py index f1ee992..bc837da 100755 --- a/pipeline/lib/build-fingerprint-index.py +++ b/pipeline/lib/build-fingerprint-index.py @@ -189,7 +189,7 @@ def main() -> int: elapsed = time.time() - start print(f"[index] done in {elapsed:.0f}s. wrote {done - failed} rows, {failed} failed. " f"index now has {total} entries.", flush=True) - return 0 if failed == 0 else 1 + return 0 if (done - failed) > 0 else 1 if __name__ == "__main__": diff --git a/pipeline/lib/enrich-buy-url.py b/pipeline/lib/enrich-buy-url.py index c0911c3..7d0f856 100755 --- a/pipeline/lib/enrich-buy-url.py +++ b/pipeline/lib/enrich-buy-url.py @@ -294,6 +294,9 @@ def set_flac_tag(path, name, value): # ---- AzuraCast reprocess (same as backfill-buy-url.py) ---------------------- def az_reprocess(base, key, station, host_paths, library, log): + if not base: + log(" no AzuraCast base URL configured — skipping reprocess") + return root = library.rstrip("/") + "/" want = {p[len(root):] for p in host_paths if p.startswith(root)} headers = {"X-API-Key": key, "Accept": "application/json"} @@ -365,7 +368,7 @@ def refresh_qobuz_links(flacs, args, az_key): f"{converted} {'converted' if args.apply else 'would-convert'}, " f"{unconvertible} left intact (not purchasable / unresolved)") - if args.apply and touched and az_key: + if args.apply and touched and az_key and args.azuracast_base: print("[refresh-qobuz] telling AzuraCast to reprocess touched files...") az_reprocess(args.azuracast_base, az_key, args.station, touched, args.library, lambda m: print(m)) @@ -415,6 +418,16 @@ def main(): if os.path.exists(keyfile): az_key = Path(keyfile).read_text().strip() + # Same cascade for the base URL: --azuracast-base, then env, then the file + # credential_service renders from Settings -> Credentials -> AzuraCast. + # The scheduled job (scheduler_service) invokes this script with neither + # the flag nor the env var set, so the keyfile is the only way it ever + # gets one. + if not args.azuracast_base: + basefile = f"{_ALEMBIC_CONFIG_DIR}/pipeline/azuracast/base_url" + if os.path.exists(basefile): + args.azuracast_base = Path(basefile).read_text().strip() + cascade = [s.strip() for s in args.sources.split(",") if s.strip() in SOURCES] if not cascade: sys.exit(f"[enrich] no valid sources in {args.sources!r}") @@ -518,12 +531,12 @@ def main(): f" no-match={looked - found}" + (f" upgraded={upgraded}" if upgrade_from else "")) - if args.apply and touched and az_key: + if args.apply and touched and az_key and args.azuracast_base: print("[enrich] telling AzuraCast to reprocess touched files...") az_reprocess(args.azuracast_base, az_key, args.station, touched, args.library, lambda m: print(m)) - elif args.apply and touched and not az_key: - print("[enrich] no AzuraCast key (flag/env/keyfile) — tags written; " + elif args.apply and touched and (not az_key or not args.azuracast_base): + print("[enrich] no AzuraCast key/base URL (flag/env/keyfile) — tags written; " "AzuraCast's periodic media scan will pick them up.") elif not args.apply: print("[enrich] re-run with --apply to write (key via --azuracast-key, " diff --git a/pipeline/lib/fix-track-metadata.py b/pipeline/lib/fix-track-metadata.py index edb5e66..a2e89d2 100755 --- a/pipeline/lib/fix-track-metadata.py +++ b/pipeline/lib/fix-track-metadata.py @@ -89,7 +89,8 @@ def _spotify_token(): continue k, v = line.split("=", 1) env[k] = v.strip().strip("'").strip('"') - return get_token(env["SPOTIFY_CLIENT_ID"], env["SPOTIFY_CLIENT_SECRET"]) + return get_token(env["SPOTIFY_CLIENT_ID"], env["SPOTIFY_CLIENT_SECRET"], + env.get("SPOTIFY_REFRESH_TOKEN") or None) def _spotify_get(path, token): diff --git a/pipeline/lib/manual/backfill-spotify-tags.sh b/pipeline/lib/manual/backfill-spotify-tags.sh index 37495db..7f93ea2 100755 --- a/pipeline/lib/manual/backfill-spotify-tags.sh +++ b/pipeline/lib/manual/backfill-spotify-tags.sh @@ -10,10 +10,21 @@ set -u CONFIGS="${ALEMBIC_CONFIG_DIR:-/config}/pipeline" LOG="${ALEMBIC_CONFIG_DIR:-/config}/logs/backfill-$(date +%Y%m%d-%H%M%S).log" RETAG="${PIPELINE_DIR:-/app/pipeline}/lib/spotify-retag.py" +SPOTIFY_ENV="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/_spotify.env" mkdir -p "$(dirname "$LOG")" echo "[$(date -Iseconds)] === Backfill start ===" | tee -a "$LOG" +# Spotify credentials (client id/secret, and a refresh token if an account is +# connected via /connect/spotify) are rendered once to _spotify.env as well as +# into each playlist .conf; read from the single shared file here rather than +# re-scraping every conf individually. +[ -f "$SPOTIFY_ENV" ] && source "$SPOTIFY_ENV" +if [[ -z "${SPOTIFY_CLIENT_ID:-}" || -z "${SPOTIFY_CLIENT_SECRET:-}" ]]; then + echo "[$(date -Iseconds)] ERROR: Spotify credentials not found at $SPOTIFY_ENV" | tee -a "$LOG" + exit 1 +fi + TOTAL_TRACKS=0 TOTAL_PLAYLISTS=0 @@ -22,11 +33,9 @@ for conf in "$CONFIGS"/*.conf; do playlist=$(basename "$conf" .conf) url=$(sed -n 's/^input *= *//p' "$conf" | tr -d ' ') - cid=$(sed -n 's/^spotify-id *= *//p' "$conf" | tr -d ' ') - csec=$(sed -n 's/^spotify-secret *= *//p' "$conf" | tr -d ' ') - if [[ -z "$url" || -z "$cid" || -z "$csec" ]]; then - echo "[$playlist] SKIP: missing url/creds in conf" | tee -a "$LOG" + if [[ -z "$url" ]]; then + echo "[$playlist] SKIP: missing url in conf" | tee -a "$LOG" continue fi @@ -41,7 +50,8 @@ for conf in "$CONFIGS"/*.conf; do fi echo "[$playlist] retagging $count tracks via Spotify" | tee -a "$LOG" - if echo "$paths" | SPOTIFY_CLIENT_ID="$cid" SPOTIFY_CLIENT_SECRET="$csec" \ + if echo "$paths" | SPOTIFY_CLIENT_ID="$SPOTIFY_CLIENT_ID" SPOTIFY_CLIENT_SECRET="$SPOTIFY_CLIENT_SECRET" \ + SPOTIFY_REFRESH_TOKEN="${SPOTIFY_REFRESH_TOKEN:-}" \ python3 "$RETAG" "$url" - >> "$LOG" 2>&1; then echo "[$playlist] retag OK" | tee -a "$LOG" TOTAL_PLAYLISTS=$((TOTAL_PLAYLISTS + 1)) diff --git a/pipeline/lib/spotify-genre.py b/pipeline/lib/spotify-genre.py index 9126d02..e1cf912 100755 --- a/pipeline/lib/spotify-genre.py +++ b/pipeline/lib/spotify-genre.py @@ -54,9 +54,9 @@ def _load_spotify_creds(): continue k, v = line.split("=", 1) env[k] = v.strip().strip("'").strip('"') - return env["SPOTIFY_CLIENT_ID"], env["SPOTIFY_CLIENT_SECRET"] + return env["SPOTIFY_CLIENT_ID"], env["SPOTIFY_CLIENT_SECRET"], env.get("SPOTIFY_REFRESH_TOKEN") or None -SPOTIFY_CID, SPOTIFY_CSEC = _load_spotify_creds() +SPOTIFY_CID, SPOTIFY_CSEC, SPOTIFY_REFRESH = _load_spotify_creds() WHITELIST_FILE = f"{ALEMBIC_CONFIG_DIR}/pipeline/genres-whitelist.txt" MAX_GENRES = 3 SEPARATOR = "; " @@ -76,7 +76,7 @@ def load_whitelist(): def _spotify_token(): - return get_token(SPOTIFY_CID, SPOTIFY_CSEC) + return get_token(SPOTIFY_CID, SPOTIFY_CSEC, SPOTIFY_REFRESH) # Cache: artist_name → list[str] of Spotify genres (or None if not found) diff --git a/pipeline/lib/spotify-retag.py b/pipeline/lib/spotify-retag.py index 130ccb0..ee242dc 100755 --- a/pipeline/lib/spotify-retag.py +++ b/pipeline/lib/spotify-retag.py @@ -22,6 +22,8 @@ Usage: spotify-retag.py - # read newline-separated paths from stdin Credentials are read from env vars SPOTIFY_CLIENT_ID and SPOTIFY_CLIENT_SECRET. +SPOTIFY_REFRESH_TOKEN, if set, mints a user token instead of client-credentials +(required for newly created Spotify apps -- see _spotify_auth.get_token). Matching strategy (per file): 1. Parse "Artist - Title.ext" from filename. If multiple " - " separators, @@ -223,6 +225,7 @@ def main() -> int: url, target = sys.argv[1], sys.argv[2] cid = os.environ.get("SPOTIFY_CLIENT_ID") csec = os.environ.get("SPOTIFY_CLIENT_SECRET") + refresh = os.environ.get("SPOTIFY_REFRESH_TOKEN") or None if not cid or not csec: sys.exit("SPOTIFY_CLIENT_ID and SPOTIFY_CLIENT_SECRET must be set") @@ -232,7 +235,7 @@ def main() -> int: files_iter = Path(target).rglob("*") print(f"[spotify-retag] Fetching playlist {url}") - token = get_token(cid, csec) + token = get_token(cid, csec, refresh) tracks = fetch_playlist(url, token) print(f"[spotify-retag] Got {len(tracks)} tracks from Spotify") diff --git a/tests/test_credential_service.py b/tests/test_credential_service.py index 563a6d2..03ad9f9 100644 --- a/tests/test_credential_service.py +++ b/tests/test_credential_service.py @@ -45,10 +45,37 @@ def test_render_playlist_confs_injects_creds_safely(db, config_dir): assert "pass = p'a$(id)ss" in text assert "spotify-id = CID" in text assert "spotify-secret = CSECRET" in text + # no account connected yet -- rendered blank, not left as the placeholder + assert "spotify-refresh = \n" in text # rendered config must be owner-only (holds the Soulseek password) assert stat.S_IMODE(os.stat(conf).st_mode) == 0o600 +def test_render_playlist_confs_includes_spotify_refresh_when_connected(db, config_dir): + cs.set_credentials(db, "spotify", {"client_id": "CID", "client_secret": "CSECRET", "refresh_token": "RTOK"}) + cs.set_credentials(db, "soulseek", {"username": "seek", "password": "pw"}) + pl.create(db, name="rtest2", spotify_url="https://open.spotify.com/playlist/Z") + + conf = config_dir / "pipeline" / "rtest2.conf" + text = conf.read_text() + assert "spotify-refresh = RTOK" in text + + +def test_azuracast_renders_and_clears_base_url(db, config_dir): + cs.set_credentials(db, "azuracast", {"base_url": "https://radio.example.com", "api_key": "KEY"}) + + az_dir = config_dir / "pipeline" / "azuracast" + assert (az_dir / "base_url").read_text() == "https://radio.example.com" + assert (az_dir / "api_key").read_text() == "KEY" + + cs.set_scope_enabled(db, "azuracast", False) + assert not (az_dir / "base_url").exists() + assert not (az_dir / "api_key").exists() + + cs.set_scope_enabled(db, "azuracast", True) + assert (az_dir / "base_url").read_text() == "https://radio.example.com" + + def test_bandcamp_cookie_expiry_parsing(): jar = "\n".join([ "# Netscape HTTP Cookie File",