andrew
59c369c9e2
Playlist detail page's status section (and the new two-column compare view) was never rendering: the router awaited status_service.playlist_status(), but that function is plain sync, so awaiting its dict return raised "object dict can't be used in 'await' expression" on every load, silently caught and shown as a generic fetch error. Removed the erroneous await. find-fuzzy-dupes.py's --apply and --apply-pairs both deleted by a `path::` regex query even though the beets id was already available in scope -- same mixed-path-storage issue (pre/post beets-2.11-upgrade items store absolute vs. library-relative paths) dedup-library.sh already worked around by deleting via id instead. This silently failed to match for most confirmed fuzzy-audio deletions (verified: 19 of 20 in one run). Added a delete_id column to dedup_candidates, threaded through the scan/apply pipeline, and switched both delete call sites to `id:`. Dedup scans also never checked whether a pair was already sitting in the pending list, so every re-scan (including the daily schedule) added a new row for the same unreviewed duplicate -- cleaned up 38 redundant rows already in production and added a check so future scans skip a pair that's already pending. Dashboard gets a Public IP stat card (cached 10 min, fetched via ipify) as a quick confidence check that outbound traffic is actually routed through gluetun's VPN and not the home connection.
384 lines
16 KiB
Python
Executable File
384 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
find-fuzzy-dupes.py — find acoustic duplicates in the beets library using
|
|
the existing $ALEMBIC_CONFIG_DIR/beets/fingerprints.db (Chromaprint).
|
|
|
|
Pairs whose duration differs by <=5s and Chromaprint similarity >= 0.92 are
|
|
flagged as duplicates. Groups are transitively merged. Within each group the
|
|
ranking is: FLAC > MP3/other, then largest size wins.
|
|
|
|
DRY RUN by default. Pass --apply to delete losers via `beet remove -d -f`.
|
|
|
|
Usage:
|
|
find-fuzzy-dupes.py
|
|
find-fuzzy-dupes.py --apply
|
|
find-fuzzy-dupes.py --threshold 0.95 --apply
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections import defaultdict
|
|
|
|
import numpy as np
|
|
|
|
FINGERPRINT_DB = f"{os.environ.get('ALEMBIC_CONFIG_DIR', '/config')}/beets/fingerprints.db"
|
|
SCAN_CACHE = "/tmp/find-fuzzy-dupes-scan-cache.json"
|
|
DEFAULT_THRESHOLD = 0.92
|
|
DURATION_TOLERANCE = 5 # seconds
|
|
# Library dedup compares same-track recordings — offsets are tiny. Use a
|
|
# smaller window than DJ-import (±80) so a 4k-file library scans in minutes.
|
|
MAX_OFFSET = 15
|
|
|
|
|
|
# Precomputed bitmasks for popcount (reused across compares).
|
|
_M1 = np.uint32(0x55555555)
|
|
_M2 = np.uint32(0x33333333)
|
|
_M4 = np.uint32(0x0F0F0F0F)
|
|
_H01 = np.uint32(0x01010101)
|
|
|
|
|
|
def _sim_at_offset(fp1: np.ndarray, fp2: np.ndarray, offset: int) -> float:
|
|
if offset >= 0:
|
|
n = min(fp1.size - offset, fp2.size)
|
|
if n < 100:
|
|
return 0.0
|
|
xor = np.bitwise_xor(fp1[offset : offset + n], fp2[:n])
|
|
else:
|
|
n = min(fp1.size, fp2.size + offset)
|
|
if n < 100:
|
|
return 0.0
|
|
xor = np.bitwise_xor(fp1[:n], fp2[-offset : -offset + n])
|
|
v = xor - ((xor >> 1) & _M1)
|
|
v = (v & _M2) + ((v >> 2) & _M2)
|
|
v = (v + (v >> 4)) & _M4
|
|
diff_bits = int(((v * _H01) >> 24).sum())
|
|
return 1.0 - diff_bits / (n * 32.0)
|
|
|
|
|
|
def _compare_fingerprints(fp1: np.ndarray, fp2: np.ndarray, max_offset: int = MAX_OFFSET) -> float:
|
|
if fp1.size < 50 or fp2.size < 50:
|
|
return 0.0
|
|
# Fast pre-filter: try offset 0 first. If unrelated, skip the offset
|
|
# search entirely. Same recordings cluster within a few offsets of 0,
|
|
# so a sub-0.50 base score is a definitive "not a duplicate."
|
|
base = _sim_at_offset(fp1, fp2, 0)
|
|
if base < 0.50:
|
|
return base
|
|
best = base
|
|
if best >= 0.99:
|
|
return best
|
|
for offset in range(1, max_offset + 1):
|
|
for o in (offset, -offset):
|
|
sim = _sim_at_offset(fp1, fp2, o)
|
|
if sim > best:
|
|
best = sim
|
|
if best >= 0.99:
|
|
return best
|
|
return best
|
|
|
|
|
|
_DESKTOP_RE = re.compile(r"-DESKTOP-[A-Z0-9]+", re.IGNORECASE)
|
|
_DUP_SUFFIX_RE = re.compile(r"\(\d+\)\.[a-z]+$|\.\d+\.[a-z]+$", re.IGNORECASE)
|
|
# DJ/club-friendly versions to prefer when same audio matches at different
|
|
# labellings. Bonus is small enough not to override format/size when those
|
|
# signal a real quality difference.
|
|
_EXTENDED_RE = re.compile(
|
|
r"\b(extended|club|dj edit|dj mix|long version|original mix)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def rank_file(host_path: str) -> int:
|
|
"""Lower is better. FLAC > non-FLAC; clean filename > sync-conflict
|
|
artifact; extended/club mixes beat radio edits at same format; then
|
|
larger wins. Penalty weights:
|
|
DESKTOP-suffix (OneDrive sync conflict) > (N)/.N. (Windows/beets dup)
|
|
so an exclusively-(1)-marked file beats one that's both (1) and DESKTOP."""
|
|
try:
|
|
size = os.path.getsize(host_path)
|
|
except OSError:
|
|
size = 0
|
|
ext_score = 1 if host_path.lower().endswith(".flac") else 0
|
|
desktop = 1 if _DESKTOP_RE.search(host_path) else 0
|
|
dup = 1 if _DUP_SUFFIX_RE.search(host_path) else 0
|
|
extended = 1 if _EXTENDED_RE.search(host_path) else 0
|
|
return (
|
|
10_000_000_000
|
|
- ext_score * 1_000_000_000_000
|
|
+ desktop * 500_000_000_000
|
|
+ dup * 100_000_000_000
|
|
- extended * 50_000_000_000
|
|
- size
|
|
)
|
|
|
|
|
|
class UnionFind:
|
|
def __init__(self, ids):
|
|
self.parent = {i: i for i in ids}
|
|
|
|
def find(self, x):
|
|
while self.parent[x] != x:
|
|
self.parent[x] = self.parent[self.parent[x]]
|
|
x = self.parent[x]
|
|
return x
|
|
|
|
def union(self, a, b):
|
|
ra, rb = self.find(a), self.find(b)
|
|
if ra != rb:
|
|
self.parent[ra] = rb
|
|
|
|
|
|
def _apply_pairs(pairs_file: str, emit_json: bool) -> int:
|
|
"""Apply already-confirmed keep/delete pairs (JSON lines: {"keep_path":
|
|
..., "delete_path": ..., "delete_id": ...}) without re-scanning the
|
|
library for duplicates. A full rescan recomputes Chromaprint similarity
|
|
for every pair in the library (10+ minutes on a ~4k track library,
|
|
longer whenever the scan cache is cold e.g. right after fingerprints.db
|
|
gets rewritten) -- wildly disproportionate for applying a decision a
|
|
human already reviewed. The one thing that actually needs re-checking
|
|
here is whether the keep/delete ranking flipped since confirmation
|
|
(e.g. the delete_path got upgraded to FLAC in the meantime); that's a
|
|
cheap, local, filesystem-only check via rank_file(), no fingerprinting
|
|
involved.
|
|
|
|
Deletes by beets id, not path -- see the id-vs-path comment on the
|
|
--apply loop in main() below for why a path:: query silently fails to
|
|
match for a large fraction of this library."""
|
|
pairs = []
|
|
with open(pairs_file) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
pairs.append(json.loads(line))
|
|
|
|
print(f"[fuzzy-dupes] applying {len(pairs)} pre-confirmed pair(s), no rescan")
|
|
deleted = failed = skipped = 0
|
|
for pair in pairs:
|
|
keep_path, delete_path, delete_id = pair["keep_path"], pair["delete_path"], pair.get("delete_id")
|
|
if not os.path.exists(delete_path):
|
|
print(f" SKIP (already gone) {delete_path}")
|
|
skipped += 1
|
|
continue
|
|
if not os.path.exists(keep_path) or rank_file(delete_path) < rank_file(keep_path):
|
|
print(f" SKIP (ranking flipped or keep_path missing since confirm) {delete_path}")
|
|
skipped += 1
|
|
continue
|
|
if delete_id is not None:
|
|
query = f"id:{delete_id}"
|
|
else:
|
|
# Pre-existing candidate confirmed before delete_id started being
|
|
# stored -- fall back to the old (less reliable) path query.
|
|
query = f"path::{re.escape(delete_path)}"
|
|
result = subprocess.run(
|
|
["beet", "remove", "-d", "-f", query],
|
|
capture_output=True, text=True,
|
|
)
|
|
if result.returncode != 0:
|
|
print(f" FAILED: {delete_path} — {result.stderr.strip()}", file=sys.stderr)
|
|
failed += 1
|
|
continue
|
|
print(f" DELETE {delete_path}")
|
|
deleted += 1
|
|
if emit_json:
|
|
print(json.dumps({"pass": "fuzzy_audio", "keep_path": keep_path, "delete_path": delete_path}))
|
|
print(f"[fuzzy-dupes] done. {deleted} deleted, {failed} failed, {skipped} skipped.")
|
|
return 1 if failed else 0
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--apply", action="store_true", help="actually delete losers")
|
|
ap.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD)
|
|
ap.add_argument("--duration-tol", type=int, default=DURATION_TOLERANCE)
|
|
ap.add_argument("--no-cache", action="store_true",
|
|
help="ignore the scan cache and re-fingerprint everything")
|
|
ap.add_argument("--json", action="store_true",
|
|
help="additionally emit one JSON line per candidate deletion to stdout, "
|
|
"for dedup_review_service to parse -- same convention as "
|
|
"dedup-library.sh --json. Purely additive.")
|
|
ap.add_argument("--only-paths", metavar="FILE",
|
|
help="in --apply mode, only delete a candidate if its path is listed "
|
|
"(one per line) in FILE -- same convention as dedup-library.sh. "
|
|
"Still re-scans the whole library first; prefer --apply-pairs for "
|
|
"applying already-confirmed pairs one at a time.")
|
|
ap.add_argument("--apply-pairs", metavar="FILE",
|
|
help="apply already-confirmed keep/delete pairs (JSON lines) without "
|
|
"re-scanning the library -- see _apply_pairs().")
|
|
args = ap.parse_args()
|
|
|
|
if args.apply_pairs:
|
|
return _apply_pairs(args.apply_pairs, args.json)
|
|
|
|
only_paths = None
|
|
if args.only_paths:
|
|
with open(args.only_paths) as f:
|
|
only_paths = {line.strip() for line in f if line.strip()}
|
|
|
|
if not os.path.exists(FINGERPRINT_DB):
|
|
print(f"ERROR: fingerprint index missing at {FINGERPRINT_DB}", file=sys.stderr)
|
|
print("Run build-fingerprint-index.py first.", file=sys.stderr)
|
|
return 2
|
|
|
|
conn = sqlite3.connect(f"file:{FINGERPRINT_DB}?mode=ro", uri=True)
|
|
rows = list(conn.execute("SELECT beets_id, path, duration, fingerprint FROM fingerprints"))
|
|
conn.close()
|
|
|
|
entries = []
|
|
for beets_id, path, duration, fp_csv in rows:
|
|
fp = np.fromstring(fp_csv, dtype=np.uint32, sep=",")
|
|
if fp.size == 0:
|
|
continue
|
|
if not os.path.exists(path):
|
|
continue
|
|
entries.append((beets_id, path, duration, fp))
|
|
print(f"[fuzzy-dupes] loaded {len(entries)} fingerprints (skipped missing/empty)")
|
|
|
|
# Bucket by duration to make pairwise scan O(N * window) instead of O(N^2).
|
|
by_dur = defaultdict(list)
|
|
for idx, (_, _, dur, _) in enumerate(entries):
|
|
by_dur[dur].append(idx)
|
|
|
|
# Try cache first. Cache key = (fingerprint mtime, threshold, duration_tol).
|
|
fp_mtime = os.path.getmtime(FINGERPRINT_DB)
|
|
cache_key = {"fp_mtime": fp_mtime, "threshold": args.threshold,
|
|
"duration_tol": args.duration_tol, "n_entries": len(entries)}
|
|
pair_scores: dict[tuple[int, int], float] = {}
|
|
if not args.no_cache and os.path.exists(SCAN_CACHE):
|
|
try:
|
|
with open(SCAN_CACHE) as f:
|
|
cache = json.load(f)
|
|
if cache.get("key") == cache_key:
|
|
pair_scores = {tuple(map(int, k.split(","))): v
|
|
for k, v in cache["pairs"].items()}
|
|
print(f"[fuzzy-dupes] loaded {len(pair_scores)} pair scores from cache "
|
|
f"({SCAN_CACHE})")
|
|
except (OSError, json.JSONDecodeError, KeyError):
|
|
pair_scores = {}
|
|
|
|
if not pair_scores:
|
|
start = time.time()
|
|
compared = 0
|
|
matched = 0
|
|
for i, (_, path_i, dur_i, fp_i) in enumerate(entries):
|
|
for d in range(dur_i - args.duration_tol, dur_i + args.duration_tol + 1):
|
|
for j in by_dur.get(d, []):
|
|
if j <= i:
|
|
continue
|
|
compared += 1
|
|
fp_j = entries[j][3]
|
|
score = _compare_fingerprints(fp_i, fp_j)
|
|
if score >= args.threshold:
|
|
pair_scores[(i, j)] = score
|
|
matched += 1
|
|
if (i + 1) % 250 == 0:
|
|
elapsed = time.time() - start
|
|
rate = (i + 1) / elapsed if elapsed > 0 else 0
|
|
print(f"[fuzzy-dupes] scanned {i + 1}/{len(entries)} "
|
|
f"({rate:.1f}/s, {compared} compared, {matched} matches)", flush=True)
|
|
try:
|
|
with open(SCAN_CACHE, "w") as f:
|
|
json.dump({
|
|
"key": cache_key,
|
|
"pairs": {f"{i},{j}": s for (i, j), s in pair_scores.items()},
|
|
}, f)
|
|
print(f"[fuzzy-dupes] cached {len(pair_scores)} pair scores → {SCAN_CACHE}")
|
|
except OSError as e:
|
|
print(f"[fuzzy-dupes] WARN: could not write cache: {e}", file=sys.stderr)
|
|
|
|
uf = UnionFind(range(len(entries)))
|
|
for (i, j) in pair_scores:
|
|
uf.union(i, j)
|
|
|
|
# Group by union-find root.
|
|
groups: dict[int, list[int]] = defaultdict(list)
|
|
for idx in range(len(entries)):
|
|
groups[uf.find(idx)].append(idx)
|
|
|
|
dup_groups = [g for g in groups.values() if len(g) >= 2]
|
|
print(f"\n[fuzzy-dupes] found {len(dup_groups)} duplicate groups "
|
|
f"covering {sum(len(g) for g in dup_groups)} files\n")
|
|
|
|
if not dup_groups:
|
|
return 0
|
|
|
|
delete_targets: list[tuple[int, str]] = []
|
|
skipped_transitive = 0
|
|
for group in sorted(dup_groups, key=lambda g: -len(g)):
|
|
ranked = sorted(group, key=lambda idx: rank_file(entries[idx][1]))
|
|
keeper_idx = ranked[0]
|
|
keeper_path = entries[keeper_idx][1]
|
|
keeper_fp = entries[keeper_idx][3]
|
|
keeper_size = os.path.getsize(keeper_path)
|
|
keeper_ext = keeper_path.rsplit(".", 1)[-1].upper()
|
|
print(f"--- group ({len(group)} files) ---")
|
|
print(f" KEEP [{keeper_ext} {keeper_size / 1024 / 1024:.1f}M] {keeper_path}")
|
|
for loser_idx in ranked[1:]:
|
|
beets_id, loser_path, _, loser_fp = entries[loser_idx]
|
|
loser_size = os.path.getsize(loser_path) if os.path.exists(loser_path) else 0
|
|
loser_ext = loser_path.rsplit(".", 1)[-1].upper()
|
|
# Verify direct similarity to the keeper. Transitive union-find can
|
|
# pull in unrelated tracks via a chain of partial matches (e.g. a
|
|
# game-OST jingle group where A~B and B~C but A and C are unrelated).
|
|
pair_key = (min(keeper_idx, loser_idx), max(keeper_idx, loser_idx))
|
|
score = pair_scores.get(pair_key)
|
|
if score is None:
|
|
score = _compare_fingerprints(keeper_fp, loser_fp)
|
|
if score < args.threshold:
|
|
print(f" SKIP [{loser_ext} {loser_size / 1024 / 1024:.1f}M sim={score:.3f} to keeper] {loser_path}")
|
|
skipped_transitive += 1
|
|
continue
|
|
print(f" DELETE [{loser_ext} {loser_size / 1024 / 1024:.1f}M sim={score:.3f}] {loser_path}")
|
|
delete_targets.append((beets_id, loser_path))
|
|
if args.json:
|
|
print(json.dumps({
|
|
"pass": "fuzzy_audio",
|
|
"keep_path": keeper_path,
|
|
"delete_path": loser_path,
|
|
"delete_id": beets_id,
|
|
"delete_size_bytes": loser_size,
|
|
"similarity": round(score, 3),
|
|
}))
|
|
print()
|
|
if skipped_transitive:
|
|
print(f"[fuzzy-dupes] skipped {skipped_transitive} transitive false-positives "
|
|
f"(chain-grouped but sim<{args.threshold} to keeper)\n")
|
|
|
|
if not args.apply:
|
|
print(f"\n[fuzzy-dupes] DRY RUN — pass --apply to delete {len(delete_targets)} files")
|
|
return 0
|
|
|
|
print(f"\n[fuzzy-dupes] APPLY: deleting {len(delete_targets)} files via beet remove -d -f")
|
|
failed = 0
|
|
skipped_unconfirmed = 0
|
|
for beets_id, path in delete_targets:
|
|
if only_paths is not None and path not in only_paths:
|
|
print(f" SKIP (not in --only-paths confirm list) {path}")
|
|
skipped_unconfirmed += 1
|
|
continue
|
|
# id, not path:: -- the beets 2.11 upgrade left the DB with mixed
|
|
# path storage (pre-upgrade items store absolute /music/... paths,
|
|
# post-upgrade imports store library-relative paths), so no single
|
|
# path query form matches both populations. Same lesson
|
|
# dedup-library.sh's process_group() already learned. Ids are
|
|
# storage-format-proof.
|
|
result = subprocess.run(
|
|
["beet", "remove", "-d", "-f", f"id:{beets_id}"],
|
|
capture_output=True, text=True,
|
|
)
|
|
if result.returncode != 0:
|
|
print(f" FAILED: {path} — {result.stderr.strip()}", file=sys.stderr)
|
|
failed += 1
|
|
deleted = len(delete_targets) - failed - skipped_unconfirmed
|
|
print(f"[fuzzy-dupes] done. {deleted} deleted, {failed} failed, {skipped_unconfirmed} skipped (unconfirmed).")
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|