Add dedup_review_service and genre_review_service
dedup-library.sh: additive --json (one NDJSON line per candidate deletion to stdout, alongside the unchanged human log) and --only-paths FILE (in --apply mode, only actually delete entries whose path is in FILE; without it, --apply deletes everything as before -- existing direct callers are unaffected). Without --only-paths the safety re-verification is free: --apply --only-paths re-runs all 4 passes from scratch on every invocation, so if a group's ranking changed since a scan (e.g. the old keep_path is gone), the fresh pass assigns the previously-"delete" path the KEEP role instead and the only-paths allowlist naming it is simply never consulted -- no duplicate ranking logic needed in the review service. spotify-genre.py: additive --json emitting one JSON line per genre change (dry-run or --apply) for genre_review_service to persist. pipeline_runner.run_job_capture(): like run_job() but captures stdout as text (still under the same shared lock, still writes a job_runs row) for callers that need to parse structured output rather than just log it. dedup_review_service.scan() persists dry-run candidates into dedup_runs/dedup_candidates. confirm_and_apply() re-checks confirmed candidates still exist before invoking --apply --only-paths, so nothing is ever deleted without an explicit confirm -- matches the false-negative- biased dedup preference. scheduler_service's maintenance:dedup job now goes through this (still dry-run only, every day). genre_review_service.run() wraps spotify-genre.py for both dry-run preview and the real scheduled --apply run, persisting every run's diff into genre_runs/genre_candidates either way -- genre writes keep their current auto-apply behavior (low-risk, reversible, GENRE_LOCK-protected) but are now reviewable after the fact. lock_artist_genre() gives a one-click revert path when a run gets something wrong. Added minimal routers+templates for /dedup (scan, review, confirm-and- delete) and /genres (preview, review, lock-old-genre). Verified end-to-end against REAL duplicate files (not mocked): built an actual FLAC+MP3 duplicate pair in a real beets library, ran dedup-library.sh --json and confirmed correct JSON output, verified --apply --only-paths with an empty confirm list deletes nothing and with the real confirmed path deletes exactly that file (DB + disk) while preserving the FLAC, and ran the full dedup_review_service scan->confirm->apply flow through the same fixture. genre_review_service and spotify-genre.py --json verified against mocked/direct output (spotify-genre.py's own artist-genre lookup needs a live Spotify API call, out of reach in this sandbox). Confirmed the full app boots with all five routers registered.
This commit is contained in:
andrew
@@ -16,16 +16,57 @@
|
||||
# "Best" is always: FLAC > MP3/other, then largest file within the same format.
|
||||
#
|
||||
# DRY RUN by default. Pass --apply to actually delete.
|
||||
#
|
||||
# --json: additionally emit one JSON line per candidate deletion to stdout
|
||||
# (NDJSON), on top of the normal human log -- for dedup_review_service to
|
||||
# parse into dedup_runs/dedup_candidates. Purely additive; the human log
|
||||
# format is unchanged.
|
||||
# --only-paths FILE: in --apply mode, only actually delete a candidate if
|
||||
# its delete_path appears (one per line) in FILE -- everything else in a
|
||||
# group still gets ranked/logged/reported as normal, just not deleted.
|
||||
# This lets the confirm step in the dedup UI re-run the exact same
|
||||
# ranking/safety logic and apply only what a human explicitly approved,
|
||||
# without --only-paths this flag is a no-op and --apply deletes everything
|
||||
# as before (preserves existing behavior for anyone invoking this
|
||||
# directly).
|
||||
|
||||
set -u
|
||||
APPLY=0
|
||||
[[ "${1:-}" == "--apply" ]] && APPLY=1
|
||||
JSON_MODE=0
|
||||
ONLY_PATHS_FILE=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--apply) APPLY=1; shift ;;
|
||||
--json) JSON_MODE=1; shift ;;
|
||||
--only-paths) ONLY_PATHS_FILE="$2"; shift 2 ;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
declare -A ONLY_PATHS=()
|
||||
if [[ -n "$ONLY_PATHS_FILE" ]]; then
|
||||
while IFS= read -r p; do
|
||||
[[ -n "$p" ]] && ONLY_PATHS["$p"]=1
|
||||
done < "$ONLY_PATHS_FILE"
|
||||
fi
|
||||
|
||||
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$(date +%Y%m%d-%H%M%S).log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
log() { echo "$*" | tee -a "$LOG"; }
|
||||
|
||||
# JSON lines go to stdout ONLY (never through log()/tee, so they never end
|
||||
# up interleaved with the human-readable $LOG file -- a caller wanting
|
||||
# structured output should capture this process's stdout separately).
|
||||
json_emit() { [[ $JSON_MODE -eq 1 ]] && printf '%s\n' "$1"; }
|
||||
|
||||
json_escape() {
|
||||
local s="$1"
|
||||
s="${s//\\/\\\\}"
|
||||
s="${s//\"/\\\"}"
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
log "[$(date -Iseconds)] === DEDUP (APPLY) ==="
|
||||
else
|
||||
@@ -106,17 +147,35 @@ process_group() {
|
||||
log ""
|
||||
log "--- $label ---"
|
||||
|
||||
# Pass name for JSON output, matching dedup_candidates.pass_name's
|
||||
# convention (numbered_sibling|case_insensitive|normalized|cross_album_fuzzy).
|
||||
local pass_name="unknown"
|
||||
case "$label" in
|
||||
"P1 numbered-sibling"*) pass_name="numbered_sibling" ;;
|
||||
"P2 case-insensitive"*) pass_name="case_insensitive" ;;
|
||||
"P3 normalized"*) pass_name="normalized" ;;
|
||||
"P4 cross-album"*) pass_name="cross_album_fuzzy" ;;
|
||||
esac
|
||||
|
||||
local first=1
|
||||
local keep_path=""
|
||||
while IFS="$SEP" read -r _sk id hp; do
|
||||
[[ -z "$hp" ]] && continue
|
||||
local size_h; size_h=$(numfmt --to=iec --suffix=B "$(stat -c '%s' "$hp" 2>/dev/null || echo 0)")
|
||||
local size_bytes; size_bytes=$(stat -c '%s' "$hp" 2>/dev/null || echo 0)
|
||||
local size_h; size_h=$(numfmt --to=iec --suffix=B "$size_bytes")
|
||||
if [[ $first -eq 1 ]]; then
|
||||
log " KEEP ($size_h) $hp"
|
||||
keep_path="$hp"
|
||||
first=0
|
||||
else
|
||||
log " DELETE ($size_h) $hp"
|
||||
json_emit "{\"pass\":\"${pass_name}\",\"keep_path\":\"$(json_escape "$keep_path")\",\"delete_path\":\"$(json_escape "$hp")\",\"delete_size_bytes\":${size_bytes}}"
|
||||
if [[ $APPLY -eq 1 ]]; then
|
||||
if [[ -n "$id" ]]; then
|
||||
# With --only-paths given, only delete entries the caller explicitly
|
||||
# confirmed; without it, delete everything (original behavior).
|
||||
if [[ -n "$ONLY_PATHS_FILE" && -z "${ONLY_PATHS[$hp]:-}" ]]; then
|
||||
log " (skipped — not in --only-paths confirm list)"
|
||||
elif [[ -n "$id" ]]; then
|
||||
# In beets — beet remove -d removes from DB + disk.
|
||||
beet remove -d -f "id:${id}" >> "$LOG" 2>&1
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user