0.6.12: Redesign the dedup review table for faster, more accurate scanning

The keep/delete columns were raw paths in a fixed-width cell: ellipsis-
truncated, full text only on hover. Slow to scan (mousing over every
row to read the song name) and, once "fixed" with a tag-derived title/
artist header in a first pass of this change, actively worse for the
thing dedup review actually needs -- confirming two files are really
the same recording. Tags can be wrong; the filename on disk can't.

New layout: each candidate gets a title row (song title, large,
unclipped -- parsed from beets' known singleton path template,
Artist/Album/Title.ext) followed by a compact detail row. The detail
row shows a color-coded format pill (FLAC/MP3/etc, matching the same
quality judgment dedup-library.sh's rank_file() already makes) and the
full relative file path, always rendered as visible text -- never
hover-only -- so a real difference in filename, album, or folder is
still plainly visible even when tags line up. Shared artist collapses
onto the title row so it isn't repeated per side; a colored left rail
marks which side survives without having to read the words.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

README.md

@@ -73,10 +73,10 @@ Optional, add these later if you want them:
 Pull the prebuilt image onto your Docker host:
 
 ```bash
-docker pull git.kretzer.club/andrew/alembic:0.6.10
+docker pull git.kretzer.club/andrew/alembic:0.6.12
 ```
 
-That is the whole install. You do not need to download the source or build anything. The `0.6.10` is the version; you can pin to it so nothing changes under you, or use `latest` to always get the newest.
+That is the whole install. You do not need to download the source or build anything. The `0.6.12` is the version; you can pin to it so nothing changes under you, or use `latest` to always get the newest.
 
 (If you would rather build it yourself from source, you can, but you do not need to.)
 
@@ -136,7 +136,7 @@ Create a file called `docker-compose.yml` on your server (put it wherever you ke
 ```yaml
 services:
   alembic:
-    image: git.kretzer.club/andrew/alembic:0.6.10
+    image: git.kretzer.club/andrew/alembic:0.6.12
     container_name: alembic
     ports:
       - "8420:8420"

app/routers/dedup.py

@@ -1,3 +1,6 @@
+import os
+import re
+
 from fastapi import APIRouter, BackgroundTasks, Depends, Request
 from fastapi.responses import RedirectResponse
 from fastapi.templating import Jinja2Templates
@@ -30,16 +33,76 @@ def _display_path(path: str) -> str:
     return path[len(_LIBRARY_PREFIX):] if path.startswith(_LIBRARY_PREFIX) else path
 
 
+# Compilation-style import paths carry a "$track " prefix on the filename
+# (see pipeline/configs/beets config.yaml paths: comp/albumtype_soundtrack);
+# singleton paths (the vast majority of this library) don't. Strip it either
+# way so the title reads clean.
+_TRACK_PREFIX_RE = re.compile(r"^\d{1,3}[\s.\-]+")
+
+# rank_file() in dedup-library.sh always ranks FLAC above every other format
+# and WAV below MP3 (a download glitch, never desirable) -- mirror that
+# judgment in the format pill's color so it reads as "this one's the keeper"
+# at a glance, not just a bare extension string.
+_FORMAT_BADGE = {"flac": "badge-success", "wav": "badge-warning"}
+
+
+def _split_display(path: str) -> dict:
+    """Break a library display path into (artist, album, title, ext).
+
+    Beets' singleton path template is always
+    %the{$albumartist}/$album/$title (pipeline/configs/beets config.yaml,
+    paths:) -- every track in this library lands at exactly that shape, so
+    the last two segments are reliably album/filename. A path that doesn't
+    fit (an unexpected root-level file) degrades to showing the raw display
+    path as the title instead of guessing at structure that isn't there.
+    """
+    display = _display_path(path)
+    parts = display.split("/")
+    if len(parts) >= 2:
+        artist, album, filename = parts[0], parts[-2], parts[-1]
+        title, ext = os.path.splitext(filename)
+        title = _TRACK_PREFIX_RE.sub("", title)
+    else:
+        artist, album = "", ""
+        title, ext = os.path.splitext(display)
+    return {
+        "path": path,
+        "display": display,
+        "artist": artist,
+        "album": album,
+        "title": title,
+        "ext": ext.lstrip(".").lower(),
+    }
+
+
+def _file_size(path: str) -> int | None:
+    try:
+        return os.path.getsize(path)
+    except OSError:
+        return None
+
+
 def _to_row(c) -> dict:
+    keep = _split_display(c.keep_path)
+    delete = _split_display(c.delete_path)
+    keep["size_bytes"] = _file_size(c.keep_path)
+    keep["badge"] = _FORMAT_BADGE.get(keep["ext"], "badge-muted")
+    delete["size_bytes"] = c.delete_size_bytes if c.delete_size_bytes is not None else _file_size(c.delete_path)
+    delete["badge"] = _FORMAT_BADGE.get(delete["ext"], "badge-muted")
     return {
         "id": c.id,
         "pass_name": c.pass_name,
         "pass_label": _pass_label(c.pass_name),
-        "keep_path": c.keep_path,
+        "keep": keep,
-        "delete_path": c.delete_path,
+        "delete": delete,
-        "keep_display": _display_path(c.keep_path),
+        # Same-tag passes (numbered_sibling/case_insensitive/normalized) share
-        "delete_display": _display_path(c.delete_path),
+        # artist/title almost always; cross_album_fuzzy and fuzzy_audio can
-        "delete_size_bytes": c.delete_size_bytes,
+        # legitimately differ (different credit, different album entirely) --
+        # the template only collapses shared context onto one line when it's
+        # actually shared, otherwise shows both sides in full.
+        "same_title": keep["title"].lower() == delete["title"].lower(),
+        "same_artist": keep["artist"].lower() == delete["artist"].lower(),
+        "same_album": keep["album"].lower() == delete["album"].lower(),
     }
 
 

app/services/dedup_review_service.py

@@ -27,6 +27,38 @@ def _script_for_pass(pass_name: str) -> str:
     return _FUZZY_SCRIPT if pass_name == _FUZZY_PASS else _SCRIPT
 
 
+def _prune_stale_pending(db) -> int:
+    """Mark pending candidates whose delete_path or keep_path no longer
+    exists as resolved, instead of leaving them to surface forever.
+
+    A pending row can go stale without ever being confirmed through the UI --
+    e.g. a later scan's numbered-twin/format-upgrade auto-apply already
+    removed one side, or upgrade-mp3-to-flac replaced it, or someone deleted
+    it by hand. confirm_and_apply() already does this "already_gone" check
+    for candidates a user explicitly acts on; this generalizes it to run at
+    the start of every scan, so the pending list reflects current reality
+    instead of stale groups that something else already resolved."""
+    now = time.time()
+    rows = db.execute(
+        select(DedupCandidate).where(
+            DedupCandidate.applied == False,  # noqa: E712
+            DedupCandidate.confirmed == False,  # noqa: E712
+            DedupCandidate.ignored == False,  # noqa: E712
+        )
+    ).scalars().all()
+    pruned = 0
+    for c in rows:
+        if not Path(c.delete_path).exists() or not Path(c.keep_path).exists():
+            c.confirmed = True
+            c.confirmed_by = "auto:stale"
+            c.confirmed_at = now
+            c.applied = True
+            pruned += 1
+    if pruned:
+        db.commit()
+    return pruned
+
+
 def _is_pair_ignored(db, path_a: str, path_b: str) -> bool:
     """True if this path pair was ever marked 'keep both', regardless of
     which path was on the keep/delete side that time -- a later scan can
@@ -78,6 +110,7 @@ async def _run_scan(
 
     db = SessionLocal()
     try:
+        _prune_stale_pending(db)
         now = time.time()
         dedup_run = DedupRun(
             started_at=job_run.started_at,
@@ -158,15 +191,19 @@ async def scan(triggered_by: str = "manual") -> DedupRun | None:
     """Dry-run dedup-library.sh --json (plus --auto-apply-numbered), persist
     every candidate deletion into a fresh dedup_runs/dedup_candidates pair.
 
-    Everything except identical numbered-sibling twins (same dir, same name,
+    Two kinds of match need no human judgment and get deleted unconditionally
-    same extension, differing only by the ".N" collision suffix) stays a
+    by dedup-library.sh itself (see is_numbered_twin/is_format_upgrade there):
-    dry-run candidate awaiting manual confirm_and_apply() below -- that's the
+    identical numbered-sibling twins (same dir, same name, same extension,
-    false-negative-biased preference for tag-based/fuzzy matches, where a
+    differing only by the ".N" collision suffix), and format upgrades within
-    wrong auto-delete could take out a genuinely different version (a
+    an EXACT tag match from Pass 2/3 (case-insensitive/normalized already
-    different album pressing, a DJ-mix edit, etc.). Numbered twins have none
+    proved same song via identical tags, so a differing extension there is
-    of that ambiguity -- it's the same download landing twice -- so
+    just "better format vs. worse"). Everything else -- including Pass 4's
-    dedup-library.sh deletes those unconditionally itself and reports them
+    cross-album fuzzy matches, where a wrong auto-delete could take out a
-    here already applied, purely for audit visibility."""
+    genuinely different version (a different album pressing, a DJ-mix edit,
+    etc.) -- stays a dry-run candidate awaiting manual confirm_and_apply()
+    below; that's the false-negative-biased preference for anything with real
+    ambiguity. Auto-applied deletions are reported here already applied,
+    purely for audit visibility."""
     return await _run_scan("dedup:scan", _SCRIPT, triggered_by, extra_args=("--auto-apply-numbered",))
 
 

app/static/style.css

@@ -475,11 +475,60 @@ tbody tr:hover { background: rgba(167, 139, 250, 0.055); }
 tbody td.actions-cell { display: flex; gap: 0.5rem; flex-wrap: wrap; }
 
 .dedup-table { table-layout: fixed; }
-.dedup-table .path-cell {
+
-  overflow: hidden;
+/* Each candidate is its own <tbody> (title row + detail row) so the pair
-  text-overflow: ellipsis;
+   highlights together on hover, and so :hover can bubble from either row
-  white-space: nowrap;
+   up to the shared group without JS. That means every detail row is now
-  max-width: 0; /* forces the cell to respect the colgroup width instead of the content's natural width */
+   a tbody's last-child, which would otherwise strip the separator between
+   one candidate and the next (the generic `tbody tr:last-child` rule above
+   assumed one shared tbody) -- restore it here, and only drop it for the
+   actual last group in the table. */
+.dedup-table .dedup-row-group:hover td { background: rgba(167, 139, 250, 0.06); }
+.dedup-table .dedup-detail-row td { border-bottom: 1px solid var(--border); }
+.dedup-table .dedup-row-group:last-of-type .dedup-detail-row td { border-bottom: none; }
+
+.dedup-title-row td { padding: 0.75rem 1.1rem 0.2rem; border-bottom: none; }
+.dedup-detail-row td { padding-top: 0.15rem; }
+
+.dedup-title {
+  font-family: var(--font-display);
+  font-size: 1.02rem;
+  font-weight: 700;
+  color: var(--fg);
+  line-height: 1.35;
+  overflow-wrap: anywhere;
+}
+.dedup-title-alt { font-weight: 400; color: var(--muted); }
+
+.dedup-context { font-size: 0.8rem; margin-top: 0.15rem; }
+
+/* A thin colored rail on each side echoes the keep/delete verdict itself --
+   green for the copy that survives, muted-pink for the one on the chopping
+   block -- so which side is which reads before you've even read the words. */
+.dedup-side {
+  display: flex;
+  align-items: center;
+  gap: 0.55rem;
+  padding-left: 0.65rem;
+  border-left: 2px solid transparent;
+  min-height: 1.6rem;
+}
+.dedup-side-keep { border-left-color: var(--status-success); }
+.dedup-side-delete { border-left-color: var(--status-danger); }
+
+.dedup-side-size { font-size: 0.78rem; white-space: nowrap; flex-shrink: 0; }
+
+/* The actual filename/path -- always visible, never truncated. Tags can be
+   wrong; this is the ground truth for judging whether two files are really
+   the same recording, so it can't be hover-only the way it was in the first
+   pass of this redesign. */
+.dedup-side-path {
+  padding-left: 0.65rem;
+  margin-top: 0.2rem;
+  font-size: 0.78rem;
+  color: var(--muted-2);
+  overflow-wrap: anywhere;
+  line-height: 1.4;
 }
 
 .empty-row td { padding: 2.5rem 1rem; text-align: center; }

app/templates/dedup/index.html

@@ -1,5 +1,65 @@
 {% extends "base.html" %}
 {% block title %}Dedup — alembic{% endblock %}
+
+{% macro candidate_rows(c, mode) %}
+<tbody class="dedup-row-group">
+  <tr class="dedup-title-row">
+    <td colspan="{{ 5 if mode == 'pending' else 4 }}">
+      {% if c.same_title %}
+      <div class="dedup-title">{{ c.keep.title }}</div>
+      {% else %}
+      <div class="dedup-title">{{ c.keep.title }} <span class="dedup-title-alt">/ {{ c.delete.title }}</span></div>
+      {% endif %}
+      {% if c.same_artist and c.same_album and c.keep.album %}
+      <div class="dedup-context muted">{{ c.keep.artist }} · {{ c.keep.album }}</div>
+      {% elif c.same_artist %}
+      <div class="dedup-context muted">{{ c.keep.artist }}</div>
+      {% else %}
+      <div class="dedup-context muted">{{ c.keep.artist }} / {{ c.delete.artist }}</div>
+      {% endif %}
+    </td>
+  </tr>
+  <tr class="dedup-detail-row">
+    {% if mode == 'pending' %}
+    <td><input type="checkbox" name="candidate_id" value="{{ c.id }}" form="bulk-delete-form"></td>
+    {% endif %}
+    <td><span class="badge {{ 'badge-warning' if c.pass_label == 'Acoustically Similar' else 'badge-info' }}" title="{{ c.pass_name }}">{{ c.pass_label }}</span></td>
+    <td>
+      <div class="dedup-side dedup-side-keep">
+        <span class="badge {{ c.keep.badge }}">{{ c.keep.ext or '?' }}</span>
+        {% if c.keep.size_bytes %}<span class="muted dedup-side-size">{{ (c.keep.size_bytes / 1024 / 1024) | round(1) }} MB</span>{% endif %}
+      </div>
+      <div class="dedup-side-path mono muted">{{ c.keep.display }}</div>
+    </td>
+    <td>
+      <div class="dedup-side dedup-side-delete">
+        <span class="badge {{ c.delete.badge }}">{{ c.delete.ext or '?' }}</span>
+        {% if c.delete.size_bytes %}<span class="muted dedup-side-size">{{ (c.delete.size_bytes / 1024 / 1024) | round(1) }} MB</span>{% endif %}
+      </div>
+      <div class="dedup-side-path mono muted">{{ c.delete.display }}</div>
+    </td>
+    <td class="actions-cell">
+      {% if mode == 'pending' %}
+      <form method="post" action="/dedup/confirm" class="inline"
+            onsubmit="return confirm('Permanently delete this file from disk?\n{{ c.delete.display }}')">
+        <input type="hidden" name="candidate_id" value="{{ c.id }}">
+        <button type="submit" class="btn btn-sm btn-danger">Delete</button>
+      </form>
+      <form method="post" action="/dedup/ignore" class="inline">
+        <input type="hidden" name="candidate_id" value="{{ c.id }}">
+        <button type="submit" class="btn btn-sm btn-ghost" title="Keep both copies and never flag this pair again">Keep both</button>
+      </form>
+      {% else %}
+      <form method="post" action="/dedup/unignore" class="inline">
+        <input type="hidden" name="candidate_id" value="{{ c.id }}">
+        <button type="submit" class="btn btn-sm btn-ghost">Un-ignore</button>
+      </form>
+      {% endif %}
+    </td>
+  </tr>
+</tbody>
+{% endmacro %}
+
 {% block content %}
 <div class="page-header">
   <div>
@@ -18,48 +78,30 @@
 {% endif %}
 
 <h2>Pending candidates</h2>
-<p class="muted" style="margin-top:-0.5rem;">Paths are relative to the library root. Hover a name for the full path.</p>
+<p class="muted" style="margin-top:-0.5rem;">Song title up top so you can scan straight down the list; the actual file path is always shown underneath each side so you can check they're really the same file before confirming.</p>
 <form id="bulk-delete-form" method="post" action="/dedup/confirm"
       onsubmit="return confirm('Permanently delete every selected file from disk? This cannot be undone.')"></form>
 <div class="table-wrap scroll">
   <table class="dedup-table">
     <colgroup>
-      <col style="width:2.2rem"><col style="width:9.5rem"><col style="width:28%">
+      <col style="width:2.2rem"><col style="width:9.5rem"><col style="width:38%">
-      <col style="width:28%"><col style="width:5rem"><col style="width:9rem">
+      <col style="width:38%"><col style="width:10rem">
     </colgroup>
     <thead>
-      <tr><th></th><th>Caught by</th><th>Keep</th><th>Delete</th><th>Size</th><th>Action</th></tr>
+      <tr><th></th><th>Caught by</th><th>Keep</th><th>Delete</th><th>Action</th></tr>
     </thead>
-    <tbody>
-      {% set pass_badge = {"File Naming": "badge-info", "Acoustically Similar": "badge-warning"} %}
     {% for c in candidates %}
-      <tr>
+    {{ candidate_rows(c, 'pending') }}
-        <td><input type="checkbox" name="candidate_id" value="{{ c.id }}" form="bulk-delete-form"></td>
-        <td><span class="badge {{ pass_badge.get(c.pass_label, 'badge-info') }}" title="{{ c.pass_name }}">{{ c.pass_label }}</span></td>
-        <td class="muted mono path-cell" title="{{ c.keep_path }}">{{ c.keep_display }}</td>
-        <td class="mono path-cell" title="{{ c.delete_path }}">{{ c.delete_display }}</td>
-        <td class="muted">{{ (c.delete_size_bytes / 1024 / 1024) | round(1) if c.delete_size_bytes else '?' }} MB</td>
-        <td class="actions-cell">
-          <form method="post" action="/dedup/confirm" class="inline"
-                onsubmit="return confirm('Permanently delete this file from disk?\n{{ c.delete_display }}')">
-            <input type="hidden" name="candidate_id" value="{{ c.id }}">
-            <button type="submit" class="btn btn-sm btn-danger">Delete</button>
-          </form>
-          <form method="post" action="/dedup/ignore" class="inline">
-            <input type="hidden" name="candidate_id" value="{{ c.id }}">
-            <button type="submit" class="btn btn-sm btn-ghost" title="Keep both copies and never flag this pair again">Keep both</button>
-          </form>
-        </td>
-      </tr>
     {% else %}
-      <tr class="empty-row"><td colspan="6">
+    <tbody>
+      <tr class="empty-row"><td colspan="5">
         <div class="empty-state">
           <span class="sparkle" style="position:relative; display:inline-block; margin-bottom:0.5rem;"></span>
           <p class="muted" style="margin:0;">No pending candidates. Run a scan.</p>
         </div>
       </td></tr>
-      {% endfor %}
     </tbody>
+    {% endfor %}
   </table>
 </div>
 {% if candidates %}
@@ -72,28 +114,15 @@
 <div class="table-wrap scroll">
   <table class="dedup-table">
     <colgroup>
-      <col style="width:9.5rem"><col style="width:33%">
+      <col style="width:9.5rem"><col style="width:40%">
-      <col style="width:33%"><col style="width:5rem"><col style="width:7rem">
+      <col style="width:40%"><col style="width:7rem">
     </colgroup>
     <thead>
-      <tr><th>Caught by</th><th>Keep</th><th>Delete</th><th>Size</th><th></th></tr>
+      <tr><th>Caught by</th><th>Keep</th><th>Delete</th><th></th></tr>
     </thead>
-    <tbody>
     {% for c in ignored %}
-      <tr>
+    {{ candidate_rows(c, 'ignored') }}
-        <td><span class="badge badge-muted" title="{{ c.pass_name }}">{{ c.pass_label }}</span></td>
-        <td class="muted mono path-cell" title="{{ c.keep_path }}">{{ c.keep_display }}</td>
-        <td class="muted mono path-cell" title="{{ c.delete_path }}">{{ c.delete_display }}</td>
-        <td class="muted">{{ (c.delete_size_bytes / 1024 / 1024) | round(1) if c.delete_size_bytes else '?' }} MB</td>
-        <td>
-          <form method="post" action="/dedup/unignore" class="inline">
-            <input type="hidden" name="candidate_id" value="{{ c.id }}">
-            <button type="submit" class="btn btn-sm btn-ghost">Un-ignore</button>
-          </form>
-        </td>
-      </tr>
     {% endfor %}
-    </tbody>
   </table>
 </div>
 {% endif %}

pipeline/lib/dedup-library.sh

@@ -40,12 +40,19 @@ while [[ $# -gt 0 ]]; do
     --apply) APPLY=1; shift ;;
     --json) JSON_MODE=1; shift ;;
     --only-paths) ONLY_PATHS_FILE="$2"; shift 2 ;;
-    # Delete Pass 1 (numbered-sibling) matches unconditionally, independent of
+    # Delete two kinds of match unconditionally, independent of
-    # --apply/--only-paths, while Passes 2-4 stay dry-run/log-only. Safe to
+    # --apply/--only-paths, while everything else stays dry-run/log-only:
-    # run unattended: a numbered-sibling pair is the SAME title, in the SAME
+    #   - numbered-sibling twins (see is_numbered_twin): same dir, same name,
-    # directory, with the SAME extension -- unlike the tag-based/fuzzy passes,
+    #     same extension, just the ".N" collision suffix -- the same download
-    # there's no fuzzy matching or cross-album ambiguity to get wrong, so it
+    #     landing twice.
-    # doesn't need a human in the loop. See dedup_review_service.scan().
+    #   - format upgrades within an exact tag match (see is_format_upgrade):
+    #     Pass 2/3 already proved same song via identical tags; a different
+    #     extension there just means "better format vs. worse", never a
+    #     different version.
+    # Excludes Pass 4 (cross-album fuzzy) entirely -- matching across
+    # different albums/directories is exactly where a different master or
+    # DJ-mix edit can share tags without being interchangeable, so it always
+    # needs a human. See dedup_review_service.scan().
     --auto-apply-numbered) AUTO_APPLY_NUMBERED=1; shift ;;
     *) shift ;;
   esac
@@ -124,6 +131,25 @@ is_numbered_twin() {
      == "$(echo "$bb" | sed -E "s/\.[0-9]+\.${ext}\$/.${ext}/")" ]]
 }
 
+# True if this pair is a same-song format upgrade found by an EXACT tag match
+# (Pass 2 case-insensitive or Pass 3 normalized -- both require identical
+# albumartist+album+title after case/punctuation folding, so "same song" is
+# already proven by the pass itself) where the two files simply have
+# different extensions. rank_file() always ranks FLAC ahead of any other
+# format regardless of size/dirty-name/etc, so within a tag-exact group a
+# cross-extension pair unconditionally means "the format-preferred copy vs.
+# a lesser one" -- no judgment call. Deliberately excludes Pass 4
+# (cross_album_fuzzy): that pass matches by artist+title-base ACROSS
+# different albums/directories, which is exactly where a different master,
+# DJ-mix edit, or reissue can share tags without being an interchangeable
+# copy -- those still need a human to look at album/context before deleting
+# either side.
+is_format_upgrade() {
+  local pass="$1" keep="$2" delete="$3"
+  [[ "$pass" == "case_insensitive" || "$pass" == "normalized" ]] || return 1
+  [[ "${keep##*.}" != "${delete##*.}" ]]
+}
+
 # Rank a file: lower = better. FLAC > MP3 > other (WAV/etc.); clean filename
 # > sync-conflict artifact (-DESKTOP-, (1), .1.); extended/club mix > radio
 # edit at same format; then larger wins.
@@ -224,7 +250,8 @@ process_group() {
       local do_delete=0 auto_applied=0
       if [[ $APPLY -eq 1 ]]; then
         do_delete=1
-      elif [[ $AUTO_APPLY_NUMBERED -eq 1 ]] && is_numbered_twin "$keep_path" "$hp"; then
+      elif [[ $AUTO_APPLY_NUMBERED -eq 1 ]] \
+        && { is_numbered_twin "$keep_path" "$hp" || is_format_upgrade "$pass_name" "$keep_path" "$hp"; }; then
         do_delete=1
         auto_applied=1
         any_auto=1