From 6114e6dc7aa79fa5372bdd3096f90ef787f2449e Mon Sep 17 00:00:00 2001 From: andrew Date: Wed, 22 Jul 2026 09:50:55 -0600 Subject: [PATCH] 0.6.11: Auto-apply exact-tag format upgrades; self-heal stale pending rows Two gaps left obvious cases stuck in the manual dedup queue: 1. Format-upgrade pairs found by Pass 2/3 (case-insensitive/normalized tag match) still required manual confirm even though the pass itself already proved same song via identical tags, and rank_file() already guarantees FLAC beats any other format regardless of size/dirty-name. A directory-casing difference (e.g. "All About This Ep" vs "...EP") meant these never matched the narrower same-directory numbered-twin rule from 0.6.10. New is_format_upgrade() auto-deletes any Pass 2/3 pair whose extensions differ, gated on pass_name so Pass 4 (cross-album fuzzy -- different masters, DJ-mix edits, genuinely ambiguous) is untouched and still requires a human. 2. A pending row can go stale without ever being confirmed -- some other action (a later auto-apply, upgrade-mp3-to-flac, a manual fix) already resolved one side of the pair -- and nothing pruned it, so an already-fixed duplicate kept surfacing in the queue indefinitely. scan() now sweeps and auto-resolves any pending candidate whose keep_path or delete_path no longer exists before persisting new ones, so the queue reflects current reality on every run instead of accumulating dead entries. Co-Authored-By: Claude Fable 5 --- README.md | 6 +-- app/services/dedup_review_service.py | 55 +++++++++++++++++++++++----- pipeline/lib/dedup-library.sh | 41 +++++++++++++++++---- 3 files changed, 83 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 117fdf1..b2c9339 100644 --- a/README.md +++ b/README.md @@ -73,10 +73,10 @@ Optional, add these later if you want them: Pull the prebuilt image onto your Docker host: ```bash -docker pull git.kretzer.club/andrew/alembic:0.6.10 +docker pull git.kretzer.club/andrew/alembic:0.6.11 ``` -That is the whole install. You do not need to download the source or build anything. The `0.6.10` is the version; you can pin to it so nothing changes under you, or use `latest` to always get the newest. +That is the whole install. You do not need to download the source or build anything. The `0.6.11` is the version; you can pin to it so nothing changes under you, or use `latest` to always get the newest. (If you would rather build it yourself from source, you can, but you do not need to.) @@ -136,7 +136,7 @@ Create a file called `docker-compose.yml` on your server (put it wherever you ke ```yaml services: alembic: - image: git.kretzer.club/andrew/alembic:0.6.10 + image: git.kretzer.club/andrew/alembic:0.6.11 container_name: alembic ports: - "8420:8420" diff --git a/app/services/dedup_review_service.py b/app/services/dedup_review_service.py index ff234d2..e96b344 100644 --- a/app/services/dedup_review_service.py +++ b/app/services/dedup_review_service.py @@ -27,6 +27,38 @@ def _script_for_pass(pass_name: str) -> str: return _FUZZY_SCRIPT if pass_name == _FUZZY_PASS else _SCRIPT +def _prune_stale_pending(db) -> int: + """Mark pending candidates whose delete_path or keep_path no longer + exists as resolved, instead of leaving them to surface forever. + + A pending row can go stale without ever being confirmed through the UI -- + e.g. a later scan's numbered-twin/format-upgrade auto-apply already + removed one side, or upgrade-mp3-to-flac replaced it, or someone deleted + it by hand. confirm_and_apply() already does this "already_gone" check + for candidates a user explicitly acts on; this generalizes it to run at + the start of every scan, so the pending list reflects current reality + instead of stale groups that something else already resolved.""" + now = time.time() + rows = db.execute( + select(DedupCandidate).where( + DedupCandidate.applied == False, # noqa: E712 + DedupCandidate.confirmed == False, # noqa: E712 + DedupCandidate.ignored == False, # noqa: E712 + ) + ).scalars().all() + pruned = 0 + for c in rows: + if not Path(c.delete_path).exists() or not Path(c.keep_path).exists(): + c.confirmed = True + c.confirmed_by = "auto:stale" + c.confirmed_at = now + c.applied = True + pruned += 1 + if pruned: + db.commit() + return pruned + + def _is_pair_ignored(db, path_a: str, path_b: str) -> bool: """True if this path pair was ever marked 'keep both', regardless of which path was on the keep/delete side that time -- a later scan can @@ -78,6 +110,7 @@ async def _run_scan( db = SessionLocal() try: + _prune_stale_pending(db) now = time.time() dedup_run = DedupRun( started_at=job_run.started_at, @@ -158,15 +191,19 @@ async def scan(triggered_by: str = "manual") -> DedupRun | None: """Dry-run dedup-library.sh --json (plus --auto-apply-numbered), persist every candidate deletion into a fresh dedup_runs/dedup_candidates pair. - Everything except identical numbered-sibling twins (same dir, same name, - same extension, differing only by the ".N" collision suffix) stays a - dry-run candidate awaiting manual confirm_and_apply() below -- that's the - false-negative-biased preference for tag-based/fuzzy matches, where a - wrong auto-delete could take out a genuinely different version (a - different album pressing, a DJ-mix edit, etc.). Numbered twins have none - of that ambiguity -- it's the same download landing twice -- so - dedup-library.sh deletes those unconditionally itself and reports them - here already applied, purely for audit visibility.""" + Two kinds of match need no human judgment and get deleted unconditionally + by dedup-library.sh itself (see is_numbered_twin/is_format_upgrade there): + identical numbered-sibling twins (same dir, same name, same extension, + differing only by the ".N" collision suffix), and format upgrades within + an EXACT tag match from Pass 2/3 (case-insensitive/normalized already + proved same song via identical tags, so a differing extension there is + just "better format vs. worse"). Everything else -- including Pass 4's + cross-album fuzzy matches, where a wrong auto-delete could take out a + genuinely different version (a different album pressing, a DJ-mix edit, + etc.) -- stays a dry-run candidate awaiting manual confirm_and_apply() + below; that's the false-negative-biased preference for anything with real + ambiguity. Auto-applied deletions are reported here already applied, + purely for audit visibility.""" return await _run_scan("dedup:scan", _SCRIPT, triggered_by, extra_args=("--auto-apply-numbered",)) diff --git a/pipeline/lib/dedup-library.sh b/pipeline/lib/dedup-library.sh index bbcc1f4..bd8d531 100755 --- a/pipeline/lib/dedup-library.sh +++ b/pipeline/lib/dedup-library.sh @@ -40,12 +40,19 @@ while [[ $# -gt 0 ]]; do --apply) APPLY=1; shift ;; --json) JSON_MODE=1; shift ;; --only-paths) ONLY_PATHS_FILE="$2"; shift 2 ;; - # Delete Pass 1 (numbered-sibling) matches unconditionally, independent of - # --apply/--only-paths, while Passes 2-4 stay dry-run/log-only. Safe to - # run unattended: a numbered-sibling pair is the SAME title, in the SAME - # directory, with the SAME extension -- unlike the tag-based/fuzzy passes, - # there's no fuzzy matching or cross-album ambiguity to get wrong, so it - # doesn't need a human in the loop. See dedup_review_service.scan(). + # Delete two kinds of match unconditionally, independent of + # --apply/--only-paths, while everything else stays dry-run/log-only: + # - numbered-sibling twins (see is_numbered_twin): same dir, same name, + # same extension, just the ".N" collision suffix -- the same download + # landing twice. + # - format upgrades within an exact tag match (see is_format_upgrade): + # Pass 2/3 already proved same song via identical tags; a different + # extension there just means "better format vs. worse", never a + # different version. + # Excludes Pass 4 (cross-album fuzzy) entirely -- matching across + # different albums/directories is exactly where a different master or + # DJ-mix edit can share tags without being interchangeable, so it always + # needs a human. See dedup_review_service.scan(). --auto-apply-numbered) AUTO_APPLY_NUMBERED=1; shift ;; *) shift ;; esac @@ -124,6 +131,25 @@ is_numbered_twin() { == "$(echo "$bb" | sed -E "s/\.[0-9]+\.${ext}\$/.${ext}/")" ]] } +# True if this pair is a same-song format upgrade found by an EXACT tag match +# (Pass 2 case-insensitive or Pass 3 normalized -- both require identical +# albumartist+album+title after case/punctuation folding, so "same song" is +# already proven by the pass itself) where the two files simply have +# different extensions. rank_file() always ranks FLAC ahead of any other +# format regardless of size/dirty-name/etc, so within a tag-exact group a +# cross-extension pair unconditionally means "the format-preferred copy vs. +# a lesser one" -- no judgment call. Deliberately excludes Pass 4 +# (cross_album_fuzzy): that pass matches by artist+title-base ACROSS +# different albums/directories, which is exactly where a different master, +# DJ-mix edit, or reissue can share tags without being an interchangeable +# copy -- those still need a human to look at album/context before deleting +# either side. +is_format_upgrade() { + local pass="$1" keep="$2" delete="$3" + [[ "$pass" == "case_insensitive" || "$pass" == "normalized" ]] || return 1 + [[ "${keep##*.}" != "${delete##*.}" ]] +} + # Rank a file: lower = better. FLAC > MP3 > other (WAV/etc.); clean filename # > sync-conflict artifact (-DESKTOP-, (1), .1.); extended/club mix > radio # edit at same format; then larger wins. @@ -224,7 +250,8 @@ process_group() { local do_delete=0 auto_applied=0 if [[ $APPLY -eq 1 ]]; then do_delete=1 - elif [[ $AUTO_APPLY_NUMBERED -eq 1 ]] && is_numbered_twin "$keep_path" "$hp"; then + elif [[ $AUTO_APPLY_NUMBERED -eq 1 ]] \ + && { is_numbered_twin "$keep_path" "$hp" || is_format_upgrade "$pass_name" "$keep_path" "$hp"; }; then do_delete=1 auto_applied=1 any_auto=1