andrew/alembic
1
0
Fork 0
Code Pull Requests Actions Releases 18 Wiki Activity

Fix library filter, compact dashboard/credentials UI, add fuzzy audio dedup

Browse Source 
- beets_service: filter params now use truthy checks instead of `is not
  None`, since a real <select> left on "(any)" submits an empty string,
  not an absent param -- the filter form silently matched zero rows for
  any real submission. Also match grouping tokens within "; "-joined
  multi-playlist values instead of requiring an exact string match.
- dashboard: credential status renders as compact dot indicators instead
  of badge+text chips, so long scope names (telegram) stay on one line.
- credentials page: two-column grid layout instead of one long column.
- find-fuzzy-dupes.py: add --json/--only-paths flags matching
  dedup-library.sh's convention, so it can plug into the same review
  queue.
- dedup_review_service: add scan_fuzzy() and route confirm_and_apply()
  to the correct underlying script (dedup-library.sh vs
  find-fuzzy-dupes.py) per candidate's pass_name, since tag-based passes
  miss duplicates whose tags differ even when the audio is identical
  (e.g. a remix credited to different artists between two copies).
- dedup page: add a "Scan for audio duplicates" trigger alongside the
  existing tag-based scan.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
andrew
2026-07-08 16:25:52 -06:00
parent c87ddc2649
commit 02744f6654
8 changed files with 217 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/beets_service.py
+63 -33
View File
@@ -34,6 +34,40 @@ def _row_to_dict(row: sqlite3.Row) -> dict:
return d
def _grouping_clause(grouping: str) -> tuple[str, list]:
"""grouping is a single tag OR a "; "-joined set (e.g. "techno; djstuff"
for a track that's in two playlists' libraries). Matching only the exact
combined string would silently exclude combo-tagged tracks from a
single-tag filter -- match the token in any position instead."""
return (
"(grouping = ? OR grouping LIKE ? OR grouping LIKE ? OR grouping LIKE ?)",
[grouping, f"{grouping}; %", f"%; {grouping}", f"%; {grouping}; %"],
)
def _build_where(grouping: str | None, search: str | None, format: str | None) -> tuple[str, list]:
where = []
params: list = []
# Truthy checks, not `is not None`: a <select> filter left on its
# "(any)" option submits an EMPTY STRING, not an absent param -- every
# real form submission includes it. `is not None` treated that as "filter
# to grouping/format == ''", which matches nothing and made every filter
# combination except an all-fields-filled one look completely broken.
if grouping:
clause, clause_params = _grouping_clause(grouping)
where.append(clause)
params.extend(clause_params)
if format:
where.append("format = ?")
params.append(format)
if search:
where.append("(artist LIKE ? OR title LIKE ? OR albumartist LIKE ?)")
like = f"%{search}%"
params.extend([like, like, like])
where_sql = f"WHERE {' AND '.join(where)}" if where else ""
return where_sql, params
def query_items(
grouping: str | None = None,
search: str | None = None,
@@ -48,23 +82,11 @@ def query_items(
if not db_exists():
return []
cols = ", ".join(_ITEM_COLUMNS)
where = []
params: list = []
if grouping is not None:
where.append("grouping = ?")
params.append(grouping)
if format is not None:
where.append("format = ?")
params.append(format)
if search:
where.append("(artist LIKE ? OR title LIKE ? OR albumartist LIKE ?)")
like = f"%{search}%"
params.extend([like, like, like])
where_sql = f"WHERE {' AND '.join(where)}" if where else ""
where_sql, params = _build_where(grouping, search, format)
limit_sql = ""
if limit is not None:
limit_sql = "LIMIT ? OFFSET ?"
params.extend([limit, offset])
params = params + [limit, offset]
conn = _connect()
try:
@@ -81,20 +103,7 @@ def count_items(grouping: str | None = None, search: str | None = None, format:
"""Matching row count for query_items()'s filters -- for pagination."""
if not db_exists():
return 0
where = []
params: list = []
if grouping is not None:
where.append("grouping = ?")
params.append(grouping)
if format is not None:
where.append("format = ?")
params.append(format)
if search:
where.append("(artist LIKE ? OR title LIKE ? OR albumartist LIKE ?)")
like = f"%{search}%"
params.extend([like, like, like])
where_sql = f"WHERE {' AND '.join(where)}" if where else ""
where_sql, params = _build_where(grouping, search, format)
conn = _connect()
try:
return conn.execute(f"SELECT COUNT(*) FROM items {where_sql}", params).fetchone()[0]
@@ -130,10 +139,6 @@ def stats() -> dict:
try:
total = conn.execute("SELECT COUNT(*) FROM items").fetchone()[0]
grouping_rows = conn.execute(
"SELECT DISTINCT grouping FROM items WHERE grouping IS NOT NULL AND grouping != ''"
).fetchall()
format_rows = conn.execute(
"SELECT format, COUNT(*) as n FROM items "
"WHERE format IS NOT NULL AND format != '' "
@@ -147,7 +152,7 @@ def stats() -> dict:
return {
"db_exists": True,
"total_tracks": total,
"groupings": sorted(r[0] for r in grouping_rows),
"groupings": distinct_groupings(conn),
"formats": [{"format": r["format"], "count": r["n"]} for r in format_rows],
"total_bytes": int(total_bytes),
}
@@ -155,6 +160,31 @@ def stats() -> dict:
conn.close()
def distinct_groupings(conn: sqlite3.Connection | None = None) -> list[str]:
"""Individual grouping tokens for the library page's filter dropdown --
split on "; " so a combo value like "techno; djstuff" contributes
"techno" and "djstuff" separately rather than cluttering the dropdown
with every raw combination that happens to exist in the DB."""
if not db_exists():
return []
owns_conn = conn is None
conn = conn or _connect()
try:
rows = conn.execute(
"SELECT DISTINCT grouping FROM items WHERE grouping IS NOT NULL AND grouping != ''"
).fetchall()
tokens = set()
for row in rows:
for part in row[0].split(";"):
part = part.strip()
if part:
tokens.add(part)
return sorted(tokens)
finally:
if owns_conn:
conn.close()
def distinct_formats() -> list[str]:
"""For the library page's format filter dropdown."""
if not db_exists():
app/services/dedup_review_service.py
+56 -22
View File
@@ -10,6 +10,12 @@ from app.services import pipeline_runner
from app.settings import settings
_SCRIPT = "dedup-library.sh"
_FUZZY_SCRIPT = "find-fuzzy-dupes.py"
_FUZZY_PASS = "fuzzy_audio"
def _script_for_pass(pass_name: str) -> str:
return _FUZZY_SCRIPT if pass_name == _FUZZY_PASS else _SCRIPT
def _parse_json_lines(output: str) -> list[dict]:
@@ -25,15 +31,10 @@ def _parse_json_lines(output: str) -> list[dict]:
return candidates
async def scan(triggered_by: str = "manual") -> DedupRun:
"""Dry-run dedup-library.sh --json, persist every candidate deletion
into a fresh dedup_runs/dedup_candidates pair. Never deletes anything
-- the scheduled maintenance:dedup job also only ever calls this (no
--apply), matching the false-negative-biased dedup preference; actual
deletion only ever happens through confirm_and_apply() below."""
script = str(settings.pipeline_dir / "lib" / _SCRIPT)
async def _run_scan(job_key: str, script_name: str, triggered_by: str) -> DedupRun:
script = str(settings.pipeline_dir / "lib" / script_name)
job_run, output = await pipeline_runner.run_job_capture(
"dedup:scan", [script, "--json"], triggered_by=triggered_by
job_key, [script, "--json"], triggered_by=triggered_by
)
candidates = _parse_json_lines(output)
@@ -69,19 +70,42 @@ async def scan(triggered_by: str = "manual") -> DedupRun:
db.close()
async def scan(triggered_by: str = "manual") -> DedupRun:
"""Dry-run dedup-library.sh --json, persist every candidate deletion
into a fresh dedup_runs/dedup_candidates pair. Never deletes anything
-- the scheduled maintenance:dedup job also only ever calls this (no
--apply), matching the false-negative-biased dedup preference; actual
deletion only ever happens through confirm_and_apply() below."""
return await _run_scan("dedup:scan", _SCRIPT, triggered_by)
async def scan_fuzzy(triggered_by: str = "manual") -> DedupRun:
"""Dry-run find-fuzzy-dupes.py --json. Tag-based passes (scan() above)
only catch duplicates whose artist/title tags overlap; this compares
Chromaprint audio fingerprints instead, so it also catches the same
recording filed under different tags (e.g. a remix credited to
different artists between two copies)."""
return await _run_scan("dedup:scan_fuzzy", _FUZZY_SCRIPT, triggered_by)
async def confirm_and_apply(candidate_ids: list[int], confirmed_by: str) -> DedupRun | None:
"""Apply only the confirmed candidate deletions.
Cheap pre-check here: skip anything where delete_path or keep_path no
longer exists (something already changed it since the scan). The real
re-verification of ranking happens for free inside dedup-library.sh
itself: --apply --only-paths re-runs all 4 passes from scratch and
re-verification of ranking happens for free inside the underlying
script itself: --apply --only-paths re-runs its passes from scratch and
recomputes keep/delete for every group before consulting the only-paths
allowlist, so if a group's ranking flipped since the scan (e.g. the old
keep_path is gone and delete_path is now the last copy), the script's
fresh pass will assign delete_path the KEEP role instead -- it never
reaches a DELETE branch for it, so --only-paths naming it is simply
never consulted. No duplicate ranking logic needed here.
dedup-library.sh (tag-based passes) and find-fuzzy-dupes.py (acoustic
fingerprint pass) are separate scripts, each with its own --apply
--only-paths invocation -- candidates are grouped by pass_name and each
group is applied through the script that actually produced it.
"""
db = SessionLocal()
try:
@@ -102,16 +126,26 @@ async def confirm_and_apply(candidate_ids: list[int], confirmed_by: str) -> Dedu
if not confirmed_candidates:
return None
confirm_file = settings.logs_dir / f"dedup-confirm-{int(now * 1000)}.txt"
confirm_file.parent.mkdir(parents=True, exist_ok=True)
confirm_file.write_text("\n".join(c.delete_path for c in confirmed_candidates) + "\n")
by_script: dict[str, list[DedupCandidate]] = {}
for c in confirmed_candidates:
by_script.setdefault(_script_for_pass(c.pass_name), []).append(c)
script = str(settings.pipeline_dir / "lib" / _SCRIPT)
job_run, _output = await pipeline_runner.run_job_capture(
"dedup:apply",
[script, "--apply", "--only-paths", str(confirm_file), "--json"],
triggered_by=f"manual:{confirmed_by}",
)
finished_at = now
log_paths = []
for script_name, group in by_script.items():
confirm_file = settings.logs_dir / f"dedup-confirm-{int(now * 1000)}-{script_name}.txt"
confirm_file.parent.mkdir(parents=True, exist_ok=True)
confirm_file.write_text("\n".join(c.delete_path for c in group) + "\n")
script = str(settings.pipeline_dir / "lib" / script_name)
job_run, _output = await pipeline_runner.run_job_capture(
"dedup:apply",
[script, "--apply", "--only-paths", str(confirm_file), "--json"],
triggered_by=f"manual:{confirmed_by}",
)
finished_at = job_run.finished_at
if job_run.log_path:
log_paths.append(job_run.log_path)
still_there = {c.delete_path for c in confirmed_candidates if Path(c.delete_path).exists()}
actually_deleted = 0
@@ -122,12 +156,12 @@ async def confirm_and_apply(candidate_ids: list[int], confirmed_by: str) -> Dedu
db.commit()
apply_run = DedupRun(
started_at=job_run.started_at,
finished_at=job_run.finished_at,
started_at=now,
finished_at=finished_at,
mode="apply",
deleted=actually_deleted,
kept=len(confirmed_candidates) - actually_deleted,
log_path=job_run.log_path,
log_path=";".join(log_paths) or None,
)
db.add(apply_run)
db.commit()