Add "keep both" option to dedup review
Lets a candidate be marked ignored instead of confirmed/applied; future scans skip re-flagging the same path pair once it's been ignored, since a rescan can flip which side ranks as keep/delete without changing the user's earlier decision that the pair is fine as two copies. Also adds --apply-pairs to find-fuzzy-dupes.py to apply pre-confirmed pairs without a full rescan.
This commit is contained in:
@@ -134,6 +134,54 @@ class UnionFind:
|
||||
self.parent[ra] = rb
|
||||
|
||||
|
||||
def _apply_pairs(pairs_file: str, emit_json: bool) -> int:
|
||||
"""Apply already-confirmed keep/delete pairs (JSON lines: {"keep_path":
|
||||
..., "delete_path": ...}) without re-scanning the library for
|
||||
duplicates. A full rescan recomputes Chromaprint similarity for every
|
||||
pair in the library (10+ minutes on a ~4k track library, longer
|
||||
whenever the scan cache is cold e.g. right after fingerprints.db gets
|
||||
rewritten) -- wildly disproportionate for applying a decision a human
|
||||
already reviewed. The one thing that actually needs re-checking here
|
||||
is whether the keep/delete ranking flipped since confirmation (e.g. the
|
||||
delete_path got upgraded to FLAC in the meantime); that's a cheap,
|
||||
local, filesystem-only check via rank_file(), no fingerprinting
|
||||
involved."""
|
||||
pairs = []
|
||||
with open(pairs_file) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
pairs.append(json.loads(line))
|
||||
|
||||
print(f"[fuzzy-dupes] applying {len(pairs)} pre-confirmed pair(s), no rescan")
|
||||
deleted = failed = skipped = 0
|
||||
for pair in pairs:
|
||||
keep_path, delete_path = pair["keep_path"], pair["delete_path"]
|
||||
if not os.path.exists(delete_path):
|
||||
print(f" SKIP (already gone) {delete_path}")
|
||||
skipped += 1
|
||||
continue
|
||||
if not os.path.exists(keep_path) or rank_file(delete_path) < rank_file(keep_path):
|
||||
print(f" SKIP (ranking flipped or keep_path missing since confirm) {delete_path}")
|
||||
skipped += 1
|
||||
continue
|
||||
escaped = re.escape(delete_path)
|
||||
result = subprocess.run(
|
||||
["beet", "remove", "-d", "-f", f"path::{escaped}"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f" FAILED: {delete_path} — {result.stderr.strip()}", file=sys.stderr)
|
||||
failed += 1
|
||||
continue
|
||||
print(f" DELETE {delete_path}")
|
||||
deleted += 1
|
||||
if emit_json:
|
||||
print(json.dumps({"pass": "fuzzy_audio", "keep_path": keep_path, "delete_path": delete_path}))
|
||||
print(f"[fuzzy-dupes] done. {deleted} deleted, {failed} failed, {skipped} skipped.")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--apply", action="store_true", help="actually delete losers")
|
||||
@@ -147,9 +195,17 @@ def main() -> int:
|
||||
"dedup-library.sh --json. Purely additive.")
|
||||
ap.add_argument("--only-paths", metavar="FILE",
|
||||
help="in --apply mode, only delete a candidate if its path is listed "
|
||||
"(one per line) in FILE -- same convention as dedup-library.sh.")
|
||||
"(one per line) in FILE -- same convention as dedup-library.sh. "
|
||||
"Still re-scans the whole library first; prefer --apply-pairs for "
|
||||
"applying already-confirmed pairs one at a time.")
|
||||
ap.add_argument("--apply-pairs", metavar="FILE",
|
||||
help="apply already-confirmed keep/delete pairs (JSON lines) without "
|
||||
"re-scanning the library -- see _apply_pairs().")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.apply_pairs:
|
||||
return _apply_pairs(args.apply_pairs, args.json)
|
||||
|
||||
only_paths = None
|
||||
if args.only_paths:
|
||||
with open(args.only_paths) as f:
|
||||
|
||||
Reference in New Issue
Block a user