Add "keep both" option to dedup review
Lets a candidate be marked ignored instead of confirmed/applied; future scans skip re-flagging the same path pair once it's been ignored, since a rescan can flip which side ranks as keep/delete without changing the user's earlier decision that the pair is fine as two copies. Also adds --apply-pairs to find-fuzzy-dupes.py to apply pre-confirmed pairs without a full rescan.
This commit is contained in:
@@ -2,7 +2,7 @@ import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.models import DedupCandidate, DedupRun
|
||||
@@ -18,6 +18,21 @@ def _script_for_pass(pass_name: str) -> str:
|
||||
return _FUZZY_SCRIPT if pass_name == _FUZZY_PASS else _SCRIPT
|
||||
|
||||
|
||||
def _is_pair_ignored(db, path_a: str, path_b: str) -> bool:
|
||||
"""True if this path pair was ever marked 'keep both', regardless of
|
||||
which path was on the keep/delete side that time -- a later scan can
|
||||
flip the ranking (e.g. file sizes changed) without changing the fact
|
||||
that the user already decided this pair is fine as two copies."""
|
||||
query = select(DedupCandidate.id).where(
|
||||
DedupCandidate.ignored == True, # noqa: E712
|
||||
or_(
|
||||
(DedupCandidate.keep_path == path_a) & (DedupCandidate.delete_path == path_b),
|
||||
(DedupCandidate.keep_path == path_b) & (DedupCandidate.delete_path == path_a),
|
||||
),
|
||||
).limit(1)
|
||||
return db.execute(query).first() is not None
|
||||
|
||||
|
||||
def _parse_json_lines(output: str) -> list[dict]:
|
||||
candidates = []
|
||||
for line in output.splitlines():
|
||||
@@ -53,7 +68,11 @@ async def _run_scan(job_key: str, script_name: str, triggered_by: str) -> DedupR
|
||||
db.commit()
|
||||
db.refresh(dedup_run)
|
||||
|
||||
skipped_ignored = 0
|
||||
for c in candidates:
|
||||
if _is_pair_ignored(db, c["keep_path"], c["delete_path"]):
|
||||
skipped_ignored += 1
|
||||
continue
|
||||
db.add(
|
||||
DedupCandidate(
|
||||
dedup_run_id=dedup_run.id,
|
||||
@@ -65,6 +84,10 @@ async def _run_scan(job_key: str, script_name: str, triggered_by: str) -> DedupR
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dedup_run)
|
||||
if skipped_ignored:
|
||||
dedup_run.kept = (dedup_run.kept or 0) + skipped_ignored
|
||||
db.commit()
|
||||
db.refresh(dedup_run)
|
||||
return dedup_run
|
||||
finally:
|
||||
db.close()
|
||||
@@ -177,9 +200,58 @@ def list_pending_candidates(dedup_run_id: int | None = None) -> list[DedupCandid
|
||||
query = select(DedupCandidate).where(
|
||||
DedupCandidate.applied == False, # noqa: E712
|
||||
DedupCandidate.confirmed == False, # noqa: E712
|
||||
DedupCandidate.ignored == False, # noqa: E712
|
||||
)
|
||||
if dedup_run_id is not None:
|
||||
query = query.where(DedupCandidate.dedup_run_id == dedup_run_id)
|
||||
return list(db.execute(query).scalars())
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def list_ignored_candidates() -> list[DedupCandidate]:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
query = select(DedupCandidate).where(DedupCandidate.ignored == True).order_by( # noqa: E712
|
||||
DedupCandidate.ignored_at.desc()
|
||||
)
|
||||
return list(db.execute(query).scalars())
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def ignore_candidate(candidate_id: int, ignored_by: str) -> DedupCandidate | None:
|
||||
"""Mark a pending candidate 'keep both' -- it drops off the pending
|
||||
list immediately, and future scans skip re-creating a candidate for
|
||||
the same path pair (see _is_pair_ignored)."""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
c = db.get(DedupCandidate, candidate_id)
|
||||
if c is None or c.applied:
|
||||
return None
|
||||
c.ignored = True
|
||||
c.ignored_by = ignored_by
|
||||
c.ignored_at = time.time()
|
||||
db.commit()
|
||||
db.refresh(c)
|
||||
return c
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def unignore_candidate(candidate_id: int) -> DedupCandidate | None:
|
||||
"""Undo a 'keep both' -- the candidate returns to the pending list and
|
||||
future scans are free to re-flag the same pair again."""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
c = db.get(DedupCandidate, candidate_id)
|
||||
if c is None:
|
||||
return None
|
||||
c.ignored = False
|
||||
c.ignored_by = None
|
||||
c.ignored_at = None
|
||||
db.commit()
|
||||
db.refresh(c)
|
||||
return c
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
Reference in New Issue
Block a user