Fix stuck dedup candidate, enrich timeout/starvation, and qobuz status error

1. enrich-buy-url.py re-queried every no-match track on every daily run (external,
   rate-limited lookups over the whole backlog), which blew past its 30-min
   timeout. Cache "tried, no match" with a BUY_URL_TRIED marker tag and skip
   those for 30 days (--force still re-checks). This is what made "Look up buy
   links" fail with TIMEOUT after 1800s.

2. Move maintenance:enrich_buy_url to run last (09:45) in the 9am block, after
   the fingerprint index (09:25) and status report (09:30). enrich starting at
   09:10 and holding the shared lock is what left "Rebuild fingerprint index"
   skipped_lock.

3. pipeline-status.sh: guard the qobuz/app_id read with a file-exists check.
   `< missing 2>/dev/null` still leaks the shell's redirection error (opened
   before 2>/dev/null applies), so the daily report logged
   "qobuz/app_id: No such file or directory" every run.

4. dedup confirm_and_apply: a candidate whose delete target no longer exists
   (already removed by an earlier dedup/upgrade/hand) was filtered out and the
   apply silently no-op'd, leaving it stuck in the queue with no way to delete
   or clear it (the 100 gecs case). Now mark such candidates applied so a Delete
   click clears them. Covered by tests/test_dedup_review.py.

Tests: 46 green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/services/dedup_review_service.py

@@ -208,15 +208,33 @@ async def confirm_and_apply(candidate_ids: list[int], confirmed_by: str) -> Dedu
     try:
         candidates = [db.get(DedupCandidate, cid) for cid in candidate_ids]
         candidates = [c for c in candidates if c is not None and not c.applied]
-        candidates = [c for c in candidates if Path(c.delete_path).exists() and Path(c.keep_path).exists()]
         if not candidates:
             return None
 
+        now = time.time()
+
+        # A candidate whose delete target no longer exists (already removed by an
+        # earlier dedup, the mp3->flac upgrade, or by hand) is effectively already
+        # resolved. Mark it applied so it clears from the pending queue, instead
+        # of the old behavior of filtering it out and silently no-oping -- which
+        # left such a stale candidate stuck in the list forever with no way to
+        # delete or clear it.
+        already_gone = [c for c in candidates if not Path(c.delete_path).exists()]
+        for c in already_gone:
+            c.confirmed = True
+            c.confirmed_by = confirmed_by
+            c.confirmed_at = now
+            c.applied = True
+        if already_gone:
+            db.commit()
+
+        # The rest need both files present for a safe delete.
+        actionable = [c for c in candidates if Path(c.delete_path).exists() and Path(c.keep_path).exists()]
+
         by_script: dict[str, list[DedupCandidate]] = {}
-        for c in candidates:
+        for c in actionable:
             by_script.setdefault(_script_for_pass(c.pass_name), []).append(c)
 
-        now = time.time()
         finished_at = now
         log_paths = []
         ran_candidates: list[DedupCandidate] = []
@@ -246,7 +264,8 @@ async def confirm_and_apply(candidate_ids: list[int], confirmed_by: str) -> Dedu
             ran_candidates.extend(group)
         db.commit()
 
-        if not ran_candidates:
+        # Nothing to report: no stale ones cleared and nothing ran.
+        if not already_gone and not ran_candidates:
             return None
 
         still_there = {c.delete_path for c in ran_candidates if Path(c.delete_path).exists()}
@@ -261,7 +280,7 @@ async def confirm_and_apply(candidate_ids: list[int], confirmed_by: str) -> Dedu
             started_at=now,
             finished_at=finished_at,
             mode="apply",
-            deleted=actually_deleted,
+            deleted=actually_deleted + len(already_gone),
             kept=len(ran_candidates) - actually_deleted,
             log_path=";".join(log_paths) or None,
         )

app/services/scheduler_service.py

@@ -130,17 +130,19 @@ MAINTENANCE_JOBS: dict[str, tuple[dict, callable]] = {
         dict(minute=5, hour=9),
         _lib("maintenance:export_laptop_playlists", "export-laptop-playlists.py"),
     ),
-    "maintenance:enrich_buy_url": (
-        dict(minute=10, hour=9),
-        # 30-min cap: this hits external buy-link APIs per track and has hung
-        # holding the pipeline lock (2026-07-09/10), starving every job after
-        # it. A timeout releases the lock so the rest of the 9am chain runs.
-        _lib("maintenance:enrich_buy_url", "enrich-buy-url.py", ["--apply"], timeout=1800),
-    ),
     "maintenance:build_fingerprint_index": (
         dict(minute=25, hour=9),
         _lib("maintenance:build_fingerprint_index", "build-fingerprint-index.py", ["--workers", "8"], timeout=3600),
     ),
+    "maintenance:enrich_buy_url": (
+        # Runs LAST in the 9am block: it hits external buy-link APIs per track
+        # and can run long, so scheduling it after the fingerprint index (09:25)
+        # and the status report (09:30) keeps it from starving them via the
+        # shared lock. The 30-min cap is a backstop; enrich-buy-url.py also now
+        # caches "tried, no match" so it stops re-querying the whole backlog.
+        dict(minute=45, hour=9),
+        _lib("maintenance:enrich_buy_url", "enrich-buy-url.py", ["--apply"], timeout=1800),
+    ),
     "maintenance:pipeline_status_report": (
         dict(minute=30, hour=9),
         # Read-only (reads logs, pings Navidrome, sends Telegram). Runs WITHOUT

pipeline/lib/enrich-buy-url.py

@@ -40,6 +40,18 @@ from pathlib import Path
 
 UA = "Mozilla/5.0 (X11; Linux x86_64) enrich-buy-url/1.0"
 BUY_URL_TAG = "COMMERCIAL_INFORMATION"
+# Marker (unix timestamp) stamped on a FLAC when a buy-link lookup found
+# nothing, so daily runs don't re-query the whole no-match backlog every time.
+# Retried after the window below; --force ignores it.
+BUY_URL_TRIED_TAG = "BUY_URL_TRIED"
+_TRIED_RETRY_DAYS = 30
+
+
+def _tried_recently(ts_str):
+    try:
+        return (time.time() - float(ts_str)) < _TRIED_RETRY_DAYS * 86400
+    except (TypeError, ValueError):
+        return False
 ITUNES_SEARCH = "https://itunes.apple.com/search"
 BC_AUTOCOMPLETE = "https://bandcamp.com/api/bcsearch_public_api/1/autocomplete_elastic"
 QOBUZ_SEARCH = "https://www.qobuz.com/api.json/0.2/track/search"
@@ -451,6 +463,14 @@ def main():
             else:
                 continue                      # already tagged, not up for upgrade
         else:
+            # Skip tracks we recently tried and found nothing for, so a daily
+            # run doesn't re-query the whole no-match backlog (external, rate-
+            # limited lookups) every time -- that churn is what pushed this job
+            # past its timeout. --force re-checks everything.
+            if not args.force:
+                tried = flac_tag(sp, BUY_URL_TRIED_TAG)
+                if tried and _tried_recently(tried):
+                    continue
             run_sources = cascade             # untagged, or --force
         artist = flac_tag(sp, "ARTIST").split(";")[0].strip() or flac_tag(sp, "ALBUMARTIST")
         title = flac_tag(sp, "TITLE")
@@ -470,6 +490,10 @@ def main():
 
         rel = sp[len(args.library):].lstrip("/")
         if not url:
+            # Record that we checked and found nothing, so the next daily run
+            # skips it (until the retry window). Apply mode only.
+            if args.apply:
+                set_flac_tag(sp, BUY_URL_TRIED_TAG, str(int(time.time())))
             continue
         found += 1
         by_source[src] += 1

pipeline/lib/pipeline-status.sh

@@ -315,8 +315,13 @@ dedup_today_status() {
   # silently drops Qobuz and falls through to iTunes.
   if [[ -f ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token ]]; then
     q_token=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/token)
-    q_appid=$(tr -d '\r\n' < ${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/app_id 2>/dev/null)
-    q_appid=${q_appid:-798273057}
+    # Guard the app_id read with a file-exists check: `< missing 2>/dev/null`
+    # still leaks the shell's own "No such file" redirection error (the redirect
+    # is opened before 2>/dev/null takes effect). Most setups have a token but
+    # no separate app_id, so this fired on every run.
+    q_appid=798273057
+    q_appid_file="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/qobuz/app_id"
+    [[ -f "$q_appid_file" ]] && q_appid=$(tr -d '\r\n' < "$q_appid_file")
     q_code=$(curl -s -o /dev/null --connect-timeout 5 --max-time 12 -A "Mozilla/5.0" \
       -H "X-App-Id: $q_appid" -H "X-User-Auth-Token: $q_token" -w '%{http_code}' \
       "https://www.qobuz.com/api.json/0.2/track/search?query=test&limit=1&app_id=$q_appid" 2>/dev/null)

tests/test_dedup_review.py

@@ -0,0 +1,57 @@
+"""confirm_and_apply: a candidate whose delete target is already gone must be
+marked applied (cleared from the queue), not silently no-op'd and left stuck.
+This is the 'can't delete this one' case that stranded a stale fuzzy candidate."""
+import asyncio
+import time
+
+from app.db import SessionLocal
+from app.models import DedupCandidate, DedupRun
+from app.services import dedup_review_service as d
+
+
+def _make_candidate(keep_path: str, delete_path: str) -> int:
+    db = SessionLocal()
+    try:
+        run = DedupRun(started_at=time.time(), mode="dry_run")
+        db.add(run)
+        db.commit()
+        db.refresh(run)
+        c = DedupCandidate(
+            dedup_run_id=run.id, pass_name="fuzzy_audio",
+            keep_path=keep_path, delete_path=delete_path, delete_id=None,
+        )
+        db.add(c)
+        db.commit()
+        db.refresh(c)
+        return c.id
+    finally:
+        db.close()
+
+
+def test_already_gone_delete_path_is_resolved(tmp_path):
+    keep = tmp_path / "keep.flac"
+    keep.write_bytes(b"x")               # keep still exists
+    gone = tmp_path / "gone.mp3"          # delete target already removed
+    cid = _make_candidate(str(keep), str(gone))
+
+    # No actionable file to delete -> no subprocess/lock; the async call resolves
+    # the stale candidate directly.
+    result = asyncio.run(d.confirm_and_apply([cid], "tester"))
+
+    assert result is not None            # a run row was produced (not a silent no-op)
+    db = SessionLocal()
+    try:
+        c = db.get(DedupCandidate, cid)
+        assert c.applied is True         # cleared from the pending queue
+    finally:
+        db.close()
+    # and it no longer appears in the pending list
+    assert cid not in [c.id for c in d.list_pending_candidates()]
+
+
+def test_ignore_then_unignore(tmp_path):
+    cid = _make_candidate(str(tmp_path / "a.flac"), str(tmp_path / "b.mp3"))
+    assert d.ignore_candidate(cid, "tester").ignored is True
+    assert cid not in [c.id for c in d.list_pending_candidates()]
+    assert d.unignore_candidate(cid).ignored is False
+    assert cid in [c.id for c in d.list_pending_candidates()]