Add "keep both" option to dedup review

Lets a candidate be marked ignored instead of confirmed/applied; future
scans skip re-flagging the same path pair once it's been ignored, since
a rescan can flip which side ranks as keep/delete without changing the
user's earlier decision that the pair is fine as two copies. Also adds
--apply-pairs to find-fuzzy-dupes.py to apply pre-confirmed pairs
without a full rescan.
This commit is contained in:
andrew
2026-07-09 08:42:14 -06:00
parent 69804a9eb2
commit 9d1eee3337
6 changed files with 215 additions and 19 deletions
+73 -1
View File
@@ -2,7 +2,7 @@ import json
import time
from pathlib import Path
from sqlalchemy import select
from sqlalchemy import or_, select
from app.db import SessionLocal
from app.models import DedupCandidate, DedupRun
@@ -18,6 +18,21 @@ def _script_for_pass(pass_name: str) -> str:
return _FUZZY_SCRIPT if pass_name == _FUZZY_PASS else _SCRIPT
def _is_pair_ignored(db, path_a: str, path_b: str) -> bool:
"""True if this path pair was ever marked 'keep both', regardless of
which path was on the keep/delete side that time -- a later scan can
flip the ranking (e.g. file sizes changed) without changing the fact
that the user already decided this pair is fine as two copies."""
query = select(DedupCandidate.id).where(
DedupCandidate.ignored == True, # noqa: E712
or_(
(DedupCandidate.keep_path == path_a) & (DedupCandidate.delete_path == path_b),
(DedupCandidate.keep_path == path_b) & (DedupCandidate.delete_path == path_a),
),
).limit(1)
return db.execute(query).first() is not None
def _parse_json_lines(output: str) -> list[dict]:
candidates = []
for line in output.splitlines():
@@ -53,7 +68,11 @@ async def _run_scan(job_key: str, script_name: str, triggered_by: str) -> DedupR
db.commit()
db.refresh(dedup_run)
skipped_ignored = 0
for c in candidates:
if _is_pair_ignored(db, c["keep_path"], c["delete_path"]):
skipped_ignored += 1
continue
db.add(
DedupCandidate(
dedup_run_id=dedup_run.id,
@@ -65,6 +84,10 @@ async def _run_scan(job_key: str, script_name: str, triggered_by: str) -> DedupR
)
db.commit()
db.refresh(dedup_run)
if skipped_ignored:
dedup_run.kept = (dedup_run.kept or 0) + skipped_ignored
db.commit()
db.refresh(dedup_run)
return dedup_run
finally:
db.close()
@@ -177,9 +200,58 @@ def list_pending_candidates(dedup_run_id: int | None = None) -> list[DedupCandid
query = select(DedupCandidate).where(
DedupCandidate.applied == False, # noqa: E712
DedupCandidate.confirmed == False, # noqa: E712
DedupCandidate.ignored == False, # noqa: E712
)
if dedup_run_id is not None:
query = query.where(DedupCandidate.dedup_run_id == dedup_run_id)
return list(db.execute(query).scalars())
finally:
db.close()
def list_ignored_candidates() -> list[DedupCandidate]:
db = SessionLocal()
try:
query = select(DedupCandidate).where(DedupCandidate.ignored == True).order_by( # noqa: E712
DedupCandidate.ignored_at.desc()
)
return list(db.execute(query).scalars())
finally:
db.close()
def ignore_candidate(candidate_id: int, ignored_by: str) -> DedupCandidate | None:
"""Mark a pending candidate 'keep both' -- it drops off the pending
list immediately, and future scans skip re-creating a candidate for
the same path pair (see _is_pair_ignored)."""
db = SessionLocal()
try:
c = db.get(DedupCandidate, candidate_id)
if c is None or c.applied:
return None
c.ignored = True
c.ignored_by = ignored_by
c.ignored_at = time.time()
db.commit()
db.refresh(c)
return c
finally:
db.close()
def unignore_candidate(candidate_id: int) -> DedupCandidate | None:
"""Undo a 'keep both' -- the candidate returns to the pending list and
future scans are free to re-flag the same pair again."""
db = SessionLocal()
try:
c = db.get(DedupCandidate, candidate_id)
if c is None:
return None
c.ignored = False
c.ignored_by = None
c.ignored_at = None
db.commit()
db.refresh(c)
return c
finally:
db.close()