alembic/app/services/dedup_review_service.py

import json
import time
from pathlib import Path

from sqlalchemy import select

from app.db import SessionLocal
from app.models import DedupCandidate, DedupRun
from app.services import pipeline_runner
from app.settings import settings

_SCRIPT = "dedup-library.sh"
_FUZZY_SCRIPT = "find-fuzzy-dupes.py"
_FUZZY_PASS = "fuzzy_audio"


def _script_for_pass(pass_name: str) -> str:
    return _FUZZY_SCRIPT if pass_name == _FUZZY_PASS else _SCRIPT


def _parse_json_lines(output: str) -> list[dict]:
    candidates = []
    for line in output.splitlines():
        line = line.strip()
        if not line.startswith("{"):
            continue
        try:
            candidates.append(json.loads(line))
        except json.JSONDecodeError:
            continue
    return candidates


async def _run_scan(job_key: str, script_name: str, triggered_by: str) -> DedupRun:
    script = str(settings.pipeline_dir / "lib" / script_name)
    job_run, output = await pipeline_runner.run_job_capture(
        job_key, [script, "--json"], triggered_by=triggered_by
    )
    candidates = _parse_json_lines(output)

    db = SessionLocal()
    try:
        dedup_run = DedupRun(
            started_at=job_run.started_at,
            finished_at=job_run.finished_at,
            mode="dry_run",
            groups_found=len({(c["pass"], c["keep_path"]) for c in candidates}),
            kept=len({(c["pass"], c["keep_path"]) for c in candidates}),
            deleted=0,
            log_path=job_run.log_path,
        )
        db.add(dedup_run)
        db.commit()
        db.refresh(dedup_run)

        for c in candidates:
            db.add(
                DedupCandidate(
                    dedup_run_id=dedup_run.id,
                    pass_name=c.get("pass", "unknown"),
                    keep_path=c["keep_path"],
                    delete_path=c["delete_path"],
                    delete_size_bytes=c.get("delete_size_bytes"),
                )
            )
        db.commit()
        db.refresh(dedup_run)
        return dedup_run
    finally:
        db.close()


async def scan(triggered_by: str = "manual") -> DedupRun:
    """Dry-run dedup-library.sh --json, persist every candidate deletion
    into a fresh dedup_runs/dedup_candidates pair. Never deletes anything
    -- the scheduled maintenance:dedup job also only ever calls this (no
    --apply), matching the false-negative-biased dedup preference; actual
    deletion only ever happens through confirm_and_apply() below."""
    return await _run_scan("dedup:scan", _SCRIPT, triggered_by)


async def scan_fuzzy(triggered_by: str = "manual") -> DedupRun:
    """Dry-run find-fuzzy-dupes.py --json. Tag-based passes (scan() above)
    only catch duplicates whose artist/title tags overlap; this compares
    Chromaprint audio fingerprints instead, so it also catches the same
    recording filed under different tags (e.g. a remix credited to
    different artists between two copies)."""
    return await _run_scan("dedup:scan_fuzzy", _FUZZY_SCRIPT, triggered_by)


async def confirm_and_apply(candidate_ids: list[int], confirmed_by: str) -> DedupRun | None:
    """Apply only the confirmed candidate deletions.

    Cheap pre-check here: skip anything where delete_path or keep_path no
    longer exists (something already changed it since the scan). The real
    re-verification of ranking happens for free inside the underlying
    script itself: --apply --only-paths re-runs its passes from scratch and
    recomputes keep/delete for every group before consulting the only-paths
    allowlist, so if a group's ranking flipped since the scan (e.g. the old
    keep_path is gone and delete_path is now the last copy), the script's
    fresh pass will assign delete_path the KEEP role instead -- it never
    reaches a DELETE branch for it, so --only-paths naming it is simply
    never consulted. No duplicate ranking logic needed here.

    dedup-library.sh (tag-based passes) and find-fuzzy-dupes.py (acoustic
    fingerprint pass) are separate scripts, each with its own --apply
    --only-paths invocation -- candidates are grouped by pass_name and each
    group is applied through the script that actually produced it.
    """
    db = SessionLocal()
    try:
        candidates = [db.get(DedupCandidate, cid) for cid in candidate_ids]
        candidates = [c for c in candidates if c is not None and not c.applied]

        now = time.time()
        confirmed_candidates = []
        for c in candidates:
            if not Path(c.delete_path).exists() or not Path(c.keep_path).exists():
                continue
            c.confirmed = True
            c.confirmed_by = confirmed_by
            c.confirmed_at = now
            confirmed_candidates.append(c)
        db.commit()

        if not confirmed_candidates:
            return None

        by_script: dict[str, list[DedupCandidate]] = {}
        for c in confirmed_candidates:
            by_script.setdefault(_script_for_pass(c.pass_name), []).append(c)

        finished_at = now
        log_paths = []
        for script_name, group in by_script.items():
            confirm_file = settings.logs_dir / f"dedup-confirm-{int(now * 1000)}-{script_name}.txt"
            confirm_file.parent.mkdir(parents=True, exist_ok=True)
            confirm_file.write_text("\n".join(c.delete_path for c in group) + "\n")

            script = str(settings.pipeline_dir / "lib" / script_name)
            job_run, _output = await pipeline_runner.run_job_capture(
                "dedup:apply",
                [script, "--apply", "--only-paths", str(confirm_file), "--json"],
                triggered_by=f"manual:{confirmed_by}",
            )
            finished_at = job_run.finished_at
            if job_run.log_path:
                log_paths.append(job_run.log_path)

        still_there = {c.delete_path for c in confirmed_candidates if Path(c.delete_path).exists()}
        actually_deleted = 0
        for c in confirmed_candidates:
            if c.delete_path not in still_there:
                c.applied = True
                actually_deleted += 1
        db.commit()

        apply_run = DedupRun(
            started_at=now,
            finished_at=finished_at,
            mode="apply",
            deleted=actually_deleted,
            kept=len(confirmed_candidates) - actually_deleted,
            log_path=";".join(log_paths) or None,
        )
        db.add(apply_run)
        db.commit()
        db.refresh(apply_run)
        return apply_run
    finally:
        db.close()


def list_pending_candidates(dedup_run_id: int | None = None) -> list[DedupCandidate]:
    db = SessionLocal()
    try:
        query = select(DedupCandidate).where(
            DedupCandidate.applied == False,  # noqa: E712
            DedupCandidate.confirmed == False,  # noqa: E712
        )
        if dedup_run_id is not None:
            query = query.where(DedupCandidate.dedup_run_id == dedup_run_id)
        return list(db.execute(query).scalars())
    finally:
        db.close()