andrew
6114e6dc7a
Two gaps left obvious cases stuck in the manual dedup queue: 1. Format-upgrade pairs found by Pass 2/3 (case-insensitive/normalized tag match) still required manual confirm even though the pass itself already proved same song via identical tags, and rank_file() already guarantees FLAC beats any other format regardless of size/dirty-name. A directory-casing difference (e.g. "All About This Ep" vs "...EP") meant these never matched the narrower same-directory numbered-twin rule from 0.6.10. New is_format_upgrade() auto-deletes any Pass 2/3 pair whose extensions differ, gated on pass_name so Pass 4 (cross-album fuzzy -- different masters, DJ-mix edits, genuinely ambiguous) is untouched and still requires a human. 2. A pending row can go stale without ever being confirmed -- some other action (a later auto-apply, upgrade-mp3-to-flac, a manual fix) already resolved one side of the pair -- and nothing pruned it, so an already-fixed duplicate kept surfacing in the queue indefinitely. scan() now sweeps and auto-resolves any pending candidate whose keep_path or delete_path no longer exists before persisting new ones, so the queue reflects current reality on every run instead of accumulating dead entries. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
654 lines
27 KiB
Bash
Executable File
654 lines
27 KiB
Bash
Executable File
#!/bin/bash
|
|
# Walk the beets library for duplicates using three passes:
|
|
#
|
|
# Pass 1 — numbered siblings: find every *.N.ext in beets where *.ext also
|
|
# exists on disk in the same directory; keep the better file.
|
|
#
|
|
# Pass 2 — case-insensitive beet-level dedup: pull all tracks from beets,
|
|
# group by lowercase (albumartist, album, title); any group with
|
|
# 2+ members is a duplicate set — keep FLAC > MP3 > largest file.
|
|
#
|
|
# Pass 3 — normalized beet-level dedup: same as Pass 2 but the key strips
|
|
# all non-alphanumeric characters before grouping. Catches dupes
|
|
# whose tags differ only by separator/punctuation/feat. credit
|
|
# ("Vol. 7" vs ", Vol. 7"; "Artist1; Artist2" vs "Artist1/Artist2").
|
|
#
|
|
# "Best" is always: FLAC > MP3/other, then largest file within the same format.
|
|
#
|
|
# DRY RUN by default. Pass --apply to actually delete.
|
|
#
|
|
# --json: additionally emit one JSON line per candidate deletion to stdout
|
|
# (NDJSON), on top of the normal human log -- for dedup_review_service to
|
|
# parse into dedup_runs/dedup_candidates. Purely additive; the human log
|
|
# format is unchanged.
|
|
# --only-paths FILE: in --apply mode, only actually delete a candidate if
|
|
# its delete_path appears (one per line) in FILE -- everything else in a
|
|
# group still gets ranked/logged/reported as normal, just not deleted.
|
|
# This lets the confirm step in the dedup UI re-run the exact same
|
|
# ranking/safety logic and apply only what a human explicitly approved,
|
|
# without --only-paths this flag is a no-op and --apply deletes everything
|
|
# as before (preserves existing behavior for anyone invoking this
|
|
# directly).
|
|
|
|
set -euo pipefail
|
|
APPLY=0
|
|
JSON_MODE=0
|
|
ONLY_PATHS_FILE=""
|
|
AUTO_APPLY_NUMBERED=0
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--apply) APPLY=1; shift ;;
|
|
--json) JSON_MODE=1; shift ;;
|
|
--only-paths) ONLY_PATHS_FILE="$2"; shift 2 ;;
|
|
# Delete two kinds of match unconditionally, independent of
|
|
# --apply/--only-paths, while everything else stays dry-run/log-only:
|
|
# - numbered-sibling twins (see is_numbered_twin): same dir, same name,
|
|
# same extension, just the ".N" collision suffix -- the same download
|
|
# landing twice.
|
|
# - format upgrades within an exact tag match (see is_format_upgrade):
|
|
# Pass 2/3 already proved same song via identical tags; a different
|
|
# extension there just means "better format vs. worse", never a
|
|
# different version.
|
|
# Excludes Pass 4 (cross-album fuzzy) entirely -- matching across
|
|
# different albums/directories is exactly where a different master or
|
|
# DJ-mix edit can share tags without being interchangeable, so it always
|
|
# needs a human. See dedup_review_service.scan().
|
|
--auto-apply-numbered) AUTO_APPLY_NUMBERED=1; shift ;;
|
|
*) shift ;;
|
|
esac
|
|
done
|
|
|
|
declare -A ONLY_PATHS=()
|
|
if [[ -n "$ONLY_PATHS_FILE" ]]; then
|
|
while IFS= read -r p; do
|
|
[[ -n "$p" ]] && ONLY_PATHS["$p"]=1
|
|
done < "$ONLY_PATHS_FILE"
|
|
fi
|
|
|
|
LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$(date +%Y%m%d-%H%M%S).log
|
|
mkdir -p "$(dirname "$LOG")"
|
|
|
|
log() { echo "$*" | tee -a "$LOG"; }
|
|
|
|
# JSON lines go to stdout ONLY (never through log()/tee, so they never end
|
|
# up interleaved with the human-readable $LOG file -- a caller wanting
|
|
# structured output should capture this process's stdout separately).
|
|
# Return 0 in non-JSON mode: this is called as a standalone statement, so under
|
|
# `set -e` a non-zero return (the default dry-run path) would abort the script.
|
|
json_emit() { [[ $JSON_MODE -eq 1 ]] || return 0; printf '%s\n' "$1"; }
|
|
|
|
json_escape() {
|
|
local s="$1"
|
|
s="${s//\\/\\\\}"
|
|
s="${s//\"/\\\"}"
|
|
printf '%s' "$s"
|
|
}
|
|
|
|
if [[ $APPLY -eq 1 ]]; then
|
|
log "[$(date -Iseconds)] === DEDUP (APPLY) ==="
|
|
else
|
|
log "[$(date -Iseconds)] === DEDUP (DRY RUN — pass --apply to delete) ==="
|
|
fi
|
|
|
|
# Counters: tracked via log grep at the end (avoids bash subshell pitfalls).
|
|
# Functions write KEEP/DELETE lines with consistent prefixes; summary greps them.
|
|
|
|
# Resolve a beets-displayed path to a real path in this container. Since the
|
|
# 2026-07-08 path rewrite, beets displays ${MUSIC_DATA_DIR:-/data/music}/Library/...
|
|
# paths that are directly usable — pass those through UNCHANGED. Only legacy
|
|
# /music/... display paths (the pre-rewrite transitional mount) still need the
|
|
# prefix swap. Blindly prepending the library dir to an already-correct path
|
|
# (the old behavior) produced /data/music/Library/data/music/Library/... —
|
|
# nonexistent, so every candidate failed the -f check and every pass reported
|
|
# 0 groups from 2026-07-08 until this fix.
|
|
host_path() {
|
|
local p="$1"
|
|
if [[ "$p" == /music/* ]]; then
|
|
echo "${MUSIC_DATA_DIR:-/data/music}/Library${p#/music}"
|
|
else
|
|
echo "$p"
|
|
fi
|
|
}
|
|
|
|
SEP=$'\x1c' # ASCII file separator — safe with any music metadata
|
|
|
|
# True if two paths are the SAME directory, SAME extension, and identical
|
|
# basenames except for a numeric ".N" infix sldl/beets inserts on a filename
|
|
# collision (e.g. "Title.flac" vs "Title.1.flac"). This is deliberately
|
|
# narrower than any tag-based pass: no fuzzy matching, no cross-album
|
|
# ambiguity, just the literal same download landing twice. Used to
|
|
# auto-delete regardless of which pass found the pair -- Pass 1 only fires
|
|
# when the canonical name is a beets ghost; the very common case where BOTH
|
|
# copies are already tracked in beets (so Pass 1 defers) surfaces instead via
|
|
# Pass 2/3's tag-based grouping, and still deserves the same free pass.
|
|
is_numbered_twin() {
|
|
local a="$1" b="$2"
|
|
[[ "${a%/*}" == "${b%/*}" ]] || return 1
|
|
local ba="${a##*/}" bb="${b##*/}"
|
|
local ext="${ba##*.}"
|
|
[[ "${ext,,}" == "${bb##*.}" ]] || return 1
|
|
[[ "$(echo "$ba" | sed -E "s/\.[0-9]+\.${ext}\$/.${ext}/")" \
|
|
== "$(echo "$bb" | sed -E "s/\.[0-9]+\.${ext}\$/.${ext}/")" ]]
|
|
}
|
|
|
|
# True if this pair is a same-song format upgrade found by an EXACT tag match
|
|
# (Pass 2 case-insensitive or Pass 3 normalized -- both require identical
|
|
# albumartist+album+title after case/punctuation folding, so "same song" is
|
|
# already proven by the pass itself) where the two files simply have
|
|
# different extensions. rank_file() always ranks FLAC ahead of any other
|
|
# format regardless of size/dirty-name/etc, so within a tag-exact group a
|
|
# cross-extension pair unconditionally means "the format-preferred copy vs.
|
|
# a lesser one" -- no judgment call. Deliberately excludes Pass 4
|
|
# (cross_album_fuzzy): that pass matches by artist+title-base ACROSS
|
|
# different albums/directories, which is exactly where a different master,
|
|
# DJ-mix edit, or reissue can share tags without being an interchangeable
|
|
# copy -- those still need a human to look at album/context before deleting
|
|
# either side.
|
|
is_format_upgrade() {
|
|
local pass="$1" keep="$2" delete="$3"
|
|
[[ "$pass" == "case_insensitive" || "$pass" == "normalized" ]] || return 1
|
|
[[ "${keep##*.}" != "${delete##*.}" ]]
|
|
}
|
|
|
|
# Rank a file: lower = better. FLAC > MP3 > other (WAV/etc.); clean filename
|
|
# > sync-conflict artifact (-DESKTOP-, (1), .1.); extended/club mix > radio
|
|
# edit at same format; then larger wins.
|
|
#
|
|
# WAV ranks below MP3 deliberately: WAV has no real tag container and in this
|
|
# library only appears as a sldl download glitch. Keeping a large WAV over a
|
|
# tagged MP3 of the same track is never what we want.
|
|
rank_file() {
|
|
local hp="$1"
|
|
local size; size=$(stat -c '%s' "$hp" 2>/dev/null || echo 0)
|
|
local ext_score=0
|
|
[[ "${hp,,}" == *.flac ]] && ext_score=2
|
|
[[ "${hp,,}" == *.mp3 ]] && ext_score=1
|
|
local dirty_score=0
|
|
# (N) copy-artifact check is capped at 2 digits: "(1).flac" is a Windows
|
|
# copy artifact, but "(2014).flac" is a title ending in a year — marking
|
|
# those dirty inverted the keep preference toward their .1.flac siblings.
|
|
if [[ "$hp" =~ -DESKTOP-[A-Z0-9]+ ]] \
|
|
|| [[ "$hp" =~ \([0-9]{1,2}\)\.[a-z]+$ ]] \
|
|
|| [[ "$hp" =~ \.[0-9]+\.[a-z]+$ ]]; then
|
|
dirty_score=1
|
|
fi
|
|
local extended_score=0
|
|
shopt -s nocasematch
|
|
if [[ "$hp" =~ (extended|club\ mix|dj\ edit|dj\ mix|long\ version|original\ mix) ]]; then
|
|
extended_score=1
|
|
fi
|
|
shopt -u nocasematch
|
|
echo $((10000000000 - ext_score * 1000000000000 + dirty_score * 100000000000 - extended_score * 50000000000 - size))
|
|
}
|
|
|
|
# process_group LABEL ENTRY [ENTRY ...]
|
|
# ENTRY is either "<beets-id>\x1c<container-path>" (a track in beets — deleted
|
|
# via `beet remove id:N`) or a bare host path (a ghost file on disk only;
|
|
# rm-only to delete).
|
|
#
|
|
# Why id queries, not path queries: the beets 2.11 upgrade (~2026-06-19) left
|
|
# the DB with MIXED path storage — pre-upgrade items store absolute
|
|
# /music/... paths, post-upgrade imports store library-relative paths. No
|
|
# single path:/path:: query form matches both populations (this is what made
|
|
# every dedup DELETE a silent no-op for two weeks). Item ids are storage-
|
|
# format-proof.
|
|
process_group() {
|
|
local label="$1"; shift
|
|
|
|
local ranked=""
|
|
local hp id entry
|
|
for entry in "$@"; do
|
|
[[ -z "$entry" ]] && continue
|
|
if [[ "$entry" == *"$SEP"* ]]; then
|
|
id="${entry%%"$SEP"*}"
|
|
hp=$(host_path "${entry#*"$SEP"}")
|
|
else
|
|
id=""
|
|
hp="$entry"
|
|
fi
|
|
[[ -f "$hp" ]] || continue
|
|
local score; score=$(rank_file "$hp")
|
|
ranked+="${score}${SEP}${id}${SEP}${hp}"$'\n'
|
|
done
|
|
|
|
# No files on disk for this group — nothing to do. Return 0 (not 1): callers
|
|
# invoke process_group as a standalone statement, so under `set -e` a non-zero
|
|
# return would abort the whole run just because one group's files were gone.
|
|
[[ -z "$ranked" ]] && return 0
|
|
|
|
log ""
|
|
log "--- $label ---"
|
|
|
|
# Pass name for JSON output, matching dedup_candidates.pass_name's
|
|
# convention (numbered_sibling|case_insensitive|normalized|cross_album_fuzzy).
|
|
local pass_name="unknown"
|
|
case "$label" in
|
|
"P1 numbered-sibling"*) pass_name="numbered_sibling" ;;
|
|
"P2 case-insensitive"*) pass_name="case_insensitive" ;;
|
|
"P3 normalized"*) pass_name="normalized" ;;
|
|
"P4 cross-album"*) pass_name="cross_album_fuzzy" ;;
|
|
esac
|
|
|
|
local first=1
|
|
local keep_path="" keep_id="" any_auto=0
|
|
while IFS="$SEP" read -r _sk id hp; do
|
|
[[ -z "$hp" ]] && continue
|
|
local size_bytes; size_bytes=$(stat -c '%s' "$hp" 2>/dev/null || echo 0)
|
|
local size_h; size_h=$(numfmt --to=iec --suffix=B "$size_bytes")
|
|
if [[ $first -eq 1 ]]; then
|
|
log " KEEP ($size_h) $hp"
|
|
keep_path="$hp"
|
|
keep_id="$id"
|
|
first=0
|
|
else
|
|
log " DELETE ($size_h) $hp"
|
|
|
|
# Auto-apply-numbered deletes numbered_sibling matches unconditionally,
|
|
# bypassing --only-paths (that gate exists only for the manual confirm
|
|
# flow, which never runs alongside this flag). All other passes stay
|
|
# dry-run/log-only here regardless.
|
|
local do_delete=0 auto_applied=0
|
|
if [[ $APPLY -eq 1 ]]; then
|
|
do_delete=1
|
|
elif [[ $AUTO_APPLY_NUMBERED -eq 1 ]] \
|
|
&& { is_numbered_twin "$keep_path" "$hp" || is_format_upgrade "$pass_name" "$keep_path" "$hp"; }; then
|
|
do_delete=1
|
|
auto_applied=1
|
|
any_auto=1
|
|
fi
|
|
local auto_json="false"; [[ $auto_applied -eq 1 ]] && auto_json="true"
|
|
json_emit "{\"pass\":\"${pass_name}\",\"keep_path\":\"$(json_escape "$keep_path")\",\"delete_path\":\"$(json_escape "$hp")\",\"delete_size_bytes\":${size_bytes},\"auto_applied\":${auto_json}}"
|
|
|
|
if [[ $do_delete -eq 1 ]]; then
|
|
# With --only-paths given, only delete entries the caller explicitly
|
|
# confirmed; without it, delete everything (original behavior).
|
|
if [[ $auto_applied -eq 0 && -n "$ONLY_PATHS_FILE" && -z "${ONLY_PATHS[$hp]:-}" ]]; then
|
|
log " (skipped — not in --only-paths confirm list)"
|
|
elif [[ -n "$id" ]]; then
|
|
# In beets — beet remove -d removes from DB + disk. Best-effort: a
|
|
# single failed removal is logged, not fatal, so the rest of the
|
|
# confirmed deletions still proceed.
|
|
beet remove -d -f "id:${id}" >> "$LOG" 2>&1 || log " WARN: beet remove id:${id} failed"
|
|
else
|
|
# Ghost file — only on disk; just rm
|
|
rm -f "$hp" 2>>"$LOG" || log " WARN: rm failed for $hp"
|
|
fi
|
|
fi
|
|
fi
|
|
done < <(echo "$ranked" | grep -v '^$' | sort -n)
|
|
|
|
# An auto-applied numbered-twin deletion can leave the numbered-named file
|
|
# as the sole survivor (it won on size despite the dirty-filename penalty).
|
|
# Rename it back to the canonical name so the library doesn't accumulate
|
|
# ".1."/".2." filenames for tracks that no longer have a duplicate. Only
|
|
# auto_applied triggers this (never a manual --only-paths confirm, which
|
|
# may leave the sibling undeleted if the user didn't confirm it -- renaming
|
|
# then would collide); only when it's actually in beets (a ghost survivor
|
|
# has nothing to move); only when the name still has the dirty infix.
|
|
if [[ $any_auto -eq 1 && -n "$keep_id" && "$keep_path" =~ \.[0-9]+\.[A-Za-z0-9]+$ ]]; then
|
|
beet move "id:${keep_id}" >> "$LOG" 2>&1 || log " WARN: beet move id:${keep_id} failed (rename survivor)"
|
|
fi
|
|
return 0 # explicit: the loop's last command status must not leak out under set -e
|
|
}
|
|
|
|
# ============================================================
|
|
# Pass 1: numbered siblings
|
|
# Find every *.N.ext in beets where *.ext also exists on disk.
|
|
# ============================================================
|
|
log ""
|
|
log "[$(date -Iseconds)] === Pass 1: numbered siblings ==="
|
|
|
|
# Dump id + display path from beets once. beets normalizes $path for display
|
|
# (real /data/music/Library/... paths since the 2026-07-08 rewrite; /music/...
|
|
# on any legacy entry) even though the DB stores a mix of absolute and
|
|
# relative — so the display paths are safe to compare/convert via host_path(),
|
|
# and the ids are what we hand to beet remove/move.
|
|
beet ls -f "\$id${SEP}\$path" 2>/dev/null > /tmp/beets-id-paths.txt
|
|
cut -d"$SEP" -f2- /tmp/beets-id-paths.txt > /tmp/beets-all-paths.txt
|
|
|
|
while IFS="$SEP" read -r numbered_id numbered_cp; do
|
|
[[ -z "$numbered_cp" ]] && continue
|
|
|
|
ext="${numbered_cp##*.}"
|
|
canonical_cp=$(echo "$numbered_cp" | sed -E "s/\.[0-9]+\.${ext}$/.${ext}/")
|
|
[[ "$canonical_cp" == "$numbered_cp" ]] && continue # no .N. found
|
|
|
|
host_numbered=$(host_path "$numbered_cp")
|
|
host_canonical=$(host_path "$canonical_cp")
|
|
|
|
[[ -f "$host_numbered" ]] || continue
|
|
[[ -f "$host_canonical" ]] || continue
|
|
|
|
# If the canonical is also in beets, Pass 2 handles this pair via beet remove -d.
|
|
# Skip here to avoid a partial rm that leaves a stale beets DB entry.
|
|
grep -qxF "$canonical_cp" /tmp/beets-all-paths.txt && continue
|
|
|
|
score_numbered=$(rank_file "$host_numbered")
|
|
score_canonical=$(rank_file "$host_canonical")
|
|
|
|
if [[ $score_canonical -le $score_numbered ]]; then
|
|
# Canonical wins: delete numbered (in beets), keep canonical (ghost)
|
|
process_group "P1 numbered-sibling: ${numbered_cp##*/Library/}" \
|
|
"$host_canonical" "${numbered_id}${SEP}${numbered_cp}"
|
|
if [[ $APPLY -eq 1 || $AUTO_APPLY_NUMBERED -eq 1 ]]; then
|
|
# canonical is now a ghost file; import it into beets
|
|
beet import -q -s "${canonical_cp}" >> "$LOG" 2>&1 || log " WARN: beet import failed for ${canonical_cp}"
|
|
fi
|
|
else
|
|
# Numbered wins (canonical is ghost): rm the ghost, then beet move to rename
|
|
# the numbered file to the canonical path (id query — see process_group).
|
|
# Safe to reach with just AUTO_APPLY_NUMBERED: this loop only ever builds
|
|
# genuine numbered-sibling pairs (see is_numbered_twin), and process_group
|
|
# above already deleted the ghost canonical before this runs, so the move
|
|
# target is guaranteed clear -- no rename collision.
|
|
process_group "P1 numbered-sibling: ${numbered_cp##*/Library/}" \
|
|
"${numbered_id}${SEP}${numbered_cp}" "$host_canonical"
|
|
if [[ $APPLY -eq 1 || $AUTO_APPLY_NUMBERED -eq 1 ]]; then
|
|
beet move "id:${numbered_id}" >> "$LOG" 2>&1 || log " WARN: beet move id:${numbered_id} failed"
|
|
fi
|
|
fi
|
|
done < <(grep -E '\.[0-9]+\.(flac|mp3|m4a|ogg|opus|wav)$' /tmp/beets-id-paths.txt)
|
|
|
|
P1_GROUPS=$(grep -c '^--- P1 ' "$LOG" 2>/dev/null; true)
|
|
log ""
|
|
log "Pass 1 complete: $P1_GROUPS numbered-sibling groups"
|
|
|
|
# ============================================================
|
|
# Pass 2: case-insensitive beet-level dedup
|
|
# Re-fetch track list so Pass 1 deletions are reflected.
|
|
# ============================================================
|
|
log ""
|
|
log "[$(date -Iseconds)] === Pass 2: case-insensitive title/album/artist dedup ==="
|
|
|
|
beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
|
|
2>/dev/null > /tmp/all-tracks.txt
|
|
|
|
gawk -F"${SEP}" -v sep="$SEP" '
|
|
# Sentinel titles carry zero identity — multiple distinct tracks can land on
|
|
# the same value when spotify-retag misses and the Soulseek source had a
|
|
# blank/placeholder title. Grouping by sentinel title = deleting different
|
|
# tracks that happened to share a placeholder. Common offenders:
|
|
# "" fully empty title
|
|
# "[unknown]" "unknown" sldl/beets fallback
|
|
# "track 1" .. "track NN" stock CD-rip placeholder
|
|
function is_sentinel_title(t, lt) {
|
|
lt = tolower(t)
|
|
gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
|
|
if (lt == "") return 1
|
|
if (lt == "[unknown]" || lt == "unknown") return 1
|
|
if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
|
|
return 0
|
|
}
|
|
{
|
|
if (is_sentinel_title($4)) next
|
|
key = tolower($2) sep tolower($3) sep tolower($4)
|
|
paths[key] = paths[key] $1 sep $5 "\n"
|
|
count[key]++
|
|
}
|
|
END {
|
|
for (k in paths) {
|
|
if (count[k] >= 2) print "===" k "\n" paths[k] "---"
|
|
}
|
|
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt
|
|
|
|
GROUP_KEY=""
|
|
GROUP_PATHS=()
|
|
|
|
flush_p2_group() {
|
|
[[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
|
|
process_group "P2 case-insensitive: $GROUP_KEY" "${GROUP_PATHS[@]}"
|
|
GROUP_PATHS=()
|
|
}
|
|
|
|
while IFS= read -r line; do
|
|
if [[ "$line" == ===* ]]; then
|
|
flush_p2_group
|
|
GROUP_KEY="${line#===}"
|
|
GROUP_PATHS=()
|
|
elif [[ "$line" == "---" ]]; then
|
|
flush_p2_group
|
|
elif [[ -n "$line" ]]; then
|
|
GROUP_PATHS+=("$line")
|
|
fi
|
|
done < /tmp/dups-grouped.txt
|
|
flush_p2_group # flush last group
|
|
|
|
P2_GROUPS=$(grep -c '^--- P2 ' "$LOG" 2>/dev/null; true)
|
|
log ""
|
|
log "Pass 2 complete: $P2_GROUPS case-insensitive dup groups"
|
|
|
|
# ============================================================
|
|
# Pass 3: normalized-key dedup (strip non-alphanumerics)
|
|
# Re-fetch track list so Pass 2 deletions are reflected.
|
|
# ============================================================
|
|
log ""
|
|
log "[$(date -Iseconds)] === Pass 3: normalized title/album/artist dedup ==="
|
|
|
|
beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
|
|
2>/dev/null > /tmp/all-tracks.txt
|
|
|
|
# Normalize: lowercase + strip everything except a-z 0-9. Group by the
|
|
# concatenation. Suppresses dupe pairs already caught by Pass 2 (same exact
|
|
# key) by also tracking the raw key per group and skipping single-key groups.
|
|
gawk -F"${SEP}" -v sep="$SEP" '
|
|
function norm(s, t) {
|
|
t = tolower(s)
|
|
gsub(/[^a-z0-9]/, "", t)
|
|
return t
|
|
}
|
|
function is_sentinel_title(t, lt) {
|
|
lt = tolower(t)
|
|
gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
|
|
if (lt == "") return 1
|
|
if (lt == "[unknown]" || lt == "unknown") return 1
|
|
if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
|
|
return 0
|
|
}
|
|
{
|
|
if (is_sentinel_title($4)) next
|
|
raw = tolower($2) sep tolower($3) sep tolower($4)
|
|
na = norm($2); nb = norm($3); nc = norm($4)
|
|
# Title is the only field guaranteed unique per track. If it normalizes
|
|
# to empty (all unicode/punctuation), refuse to group — would cause false
|
|
# positives across unrelated unicode-titled tracks. A 1-char normalized
|
|
# title is fine as long as artist+album also normalize non-empty
|
|
# (covers "ID"/"OP"/single-letter tracks; rejects all-unicode titles
|
|
# that collapse to "").
|
|
if (length(nc) == 0) next
|
|
if (length(nc) < 3 && (length(na) == 0 || length(nb) == 0)) next
|
|
k = na sep nb sep nc
|
|
paths[k] = paths[k] $1 sep $5 "\n"
|
|
count[k]++
|
|
raws[k][raw] = 1
|
|
}
|
|
END {
|
|
for (k in paths) {
|
|
if (count[k] < 2) continue
|
|
# Skip groups that collapse to a single raw key — those were Pass 2 dups.
|
|
raw_count = 0
|
|
for (r in raws[k]) raw_count++
|
|
if (raw_count < 2) continue
|
|
print "===" k "\n" paths[k] "---"
|
|
}
|
|
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt
|
|
|
|
GROUP_KEY=""
|
|
GROUP_PATHS=()
|
|
|
|
flush_p3_group() {
|
|
[[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
|
|
process_group "P3 normalized: $GROUP_KEY" "${GROUP_PATHS[@]}"
|
|
GROUP_PATHS=()
|
|
}
|
|
|
|
while IFS= read -r line; do
|
|
if [[ "$line" == ===* ]]; then
|
|
flush_p3_group
|
|
GROUP_KEY="${line#===}"
|
|
GROUP_PATHS=()
|
|
elif [[ "$line" == "---" ]]; then
|
|
flush_p3_group
|
|
elif [[ -n "$line" ]]; then
|
|
GROUP_PATHS+=("$line")
|
|
fi
|
|
done < /tmp/dups-grouped.txt
|
|
flush_p3_group # flush last group
|
|
|
|
P3_GROUPS=$(grep -c '^--- P3 ' "$LOG" 2>/dev/null; true)
|
|
log ""
|
|
log "Pass 3 complete: $P3_GROUPS normalized dup groups"
|
|
|
|
# ============================================================
|
|
# Pass 4: cross-album dedup (album-agnostic key)
|
|
# Catches cases where the same track exists once as a single and once on a
|
|
# compilation/Beatport-style album — the album field always disagrees, so
|
|
# Pass 2/3 miss them. Keys on (artist_norm, title_base_norm) where title_base
|
|
# strips trailing mix-version suffixes ("(Extended Mix)", " - Radio Edit",
|
|
# bare " Extended Mix" with double-space, etc.).
|
|
#
|
|
# Allowlist: artists who legitimately have the same track on multiple albums
|
|
# (live + studio + remix etc.) — these are SKIPPED in Pass 4. Add to
|
|
# CROSS_ALBUM_KEEP_RE or the album-pattern skip below if needed.
|
|
# ============================================================
|
|
log ""
|
|
log "[$(date -Iseconds)] === Pass 4: cross-album dedup (artist+title-base) ==="
|
|
|
|
beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
|
|
2>/dev/null > /tmp/all-tracks.txt
|
|
|
|
# Pass artist + album allowlists into awk via -v.
|
|
# The artist keep-list (artists whose same-title tracks across albums are NOT
|
|
# treated as duplicates) is user-editable config, not baked in here. Build the
|
|
# anchored regex from cross-album-keep.list: normalize each name the same way
|
|
# the awk norm() does (lowercase, letters+digits only) and OR them together.
|
|
# An empty/absent list means "protect nobody" -> use a pattern that can never
|
|
# match a normalized name (which only ever contains [a-z0-9], never '_').
|
|
CROSS_ALBUM_KEEP_FILE="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/cross-album-keep.list"
|
|
if [[ -f "$CROSS_ALBUM_KEEP_FILE" ]]; then
|
|
# `|| true`: with an all-comments file (the default), the first grep matches
|
|
# nothing and returns non-zero, which under set -o pipefail + set -e would
|
|
# otherwise abort here. An empty result is the correct "protect nobody" case.
|
|
_keep_names=$(grep -vE '^[[:space:]]*(#|$)' "$CROSS_ALBUM_KEEP_FILE" \
|
|
| tr 'A-Z' 'a-z' | sed -E 's/[^a-z0-9]//g' | grep -v '^$' | paste -sd'|' - || true)
|
|
fi
|
|
if [[ -n "${_keep_names:-}" ]]; then
|
|
CROSS_ALBUM_KEEP_RE="^(${_keep_names})\$"
|
|
else
|
|
CROSS_ALBUM_KEEP_RE='_never_'
|
|
fi
|
|
# Album-name patterns that indicate "intentional alt version" — these rows are
|
|
# filtered out before grouping, so any group needing 2+ matches will only form
|
|
# from items whose albums DON'T look like remix/live/single-version releases.
|
|
# Tagging convention: when the user has multiple legitimate versions of a
|
|
# track, the alt-version copies live on albums with explicit version markers
|
|
# in the album title ("(Extended Mix) - Single", "(Remixed)", "Live at X").
|
|
# Without this filter, Pass 4 would try to dedupe Qrion's "Proud" off the
|
|
# "I Hope It Lasts Forever (Remixed)" album, Fred again._ studio tracks
|
|
# against "Apple Music Live" recordings, etc.
|
|
ALBUM_KEEP_RE='(soundtrack|originalmotionpicture|ost|vgm|gameost|liveat|livein|livefrom|livesession|applemusiclive|applelive|bbclive|bbcsession|mtvunplugged|unplugged|concert|inconcert|extended|remix|remixed|remixes|originalmix|radioedit|radiomix|clubmix|djedit|djmix|dubmix|vocalmix|instrumentalmix|longversion|shortversion|albumversion|originalversion)'
|
|
|
|
gawk -F"${SEP}" -v sep="$SEP" \
|
|
-v artist_keep="$CROSS_ALBUM_KEEP_RE" \
|
|
-v album_keep="$ALBUM_KEEP_RE" '
|
|
function norm(s, t) {
|
|
t = tolower(s)
|
|
gsub(/[^a-z0-9]/, "", t)
|
|
return t
|
|
}
|
|
# Strip trailing mix-version suffix from title so "Foo" and "Foo (Extended Mix)"
|
|
# share the same base. Only strips generic version markers — does NOT strip
|
|
# remixer-named suffixes like "(Joris Voorn Remix)", which are different songs.
|
|
function is_sentinel_title(t, lt) {
|
|
lt = tolower(t)
|
|
gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
|
|
if (lt == "") return 1
|
|
if (lt == "[unknown]" || lt == "unknown") return 1
|
|
if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
|
|
return 0
|
|
}
|
|
function strip_mix_suffix(s, t, sfx) {
|
|
t = tolower(s)
|
|
sfx = "(extended[[:space:]]*mix|extended[[:space:]]*version|extended|" \
|
|
"radio[[:space:]]*edit|radio[[:space:]]*mix|radio[[:space:]]*version|" \
|
|
"original[[:space:]]*mix|original[[:space:]]*version|" \
|
|
"club[[:space:]]*mix|dj[[:space:]]*edit|dj[[:space:]]*mix|" \
|
|
"long[[:space:]]*version|short[[:space:]]*version|" \
|
|
"single[[:space:]]*version|album[[:space:]]*version)"
|
|
# parenthetical: " (Extended Mix)"
|
|
t = gensub("[[:space:]]*\\(" sfx "\\)[[:space:]]*$", "", "g", t)
|
|
# dash-separated: " - Extended Mix"
|
|
t = gensub("[[:space:]]*-[[:space:]]+" sfx "[[:space:]]*$", "", "g", t)
|
|
# bare trailing: " Extended Mix"
|
|
t = gensub("[[:space:]]+" sfx "[[:space:]]*$", "", "g", t)
|
|
return t
|
|
}
|
|
{
|
|
id = $1; artist = $2; album = $3; title = $4; path = $5
|
|
if (is_sentinel_title(title)) next
|
|
na = norm(artist)
|
|
if (na ~ artist_keep) next
|
|
if (norm(album) ~ album_keep) next
|
|
tb = strip_mix_suffix(title)
|
|
ntb = norm(tb)
|
|
if (length(ntb) < 4) next
|
|
k = na sep ntb
|
|
paths[k] = paths[k] id sep path "\n"
|
|
count[k]++
|
|
raws3[k][na sep norm(album) sep norm(title)] = 1
|
|
albums[k][album] = 1
|
|
}
|
|
END {
|
|
for (k in paths) {
|
|
if (count[k] < 2) continue
|
|
# Skip if all items share the same Pass 3 normalized key (Pass 3 already handled).
|
|
rcount = 0; for (r in raws3[k]) rcount++
|
|
if (rcount < 2) continue
|
|
# Require ≥2 distinct album names — otherwise Pass 3 would have grouped them.
|
|
acount = 0; for (a in albums[k]) acount++
|
|
if (acount < 2) continue
|
|
print "===" k "\n" paths[k] "---"
|
|
}
|
|
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt
|
|
|
|
GROUP_KEY=""
|
|
GROUP_PATHS=()
|
|
|
|
flush_p4_group() {
|
|
[[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
|
|
process_group "P4 cross-album: $GROUP_KEY" "${GROUP_PATHS[@]}"
|
|
GROUP_PATHS=()
|
|
}
|
|
|
|
while IFS= read -r line; do
|
|
if [[ "$line" == ===* ]]; then
|
|
flush_p4_group
|
|
GROUP_KEY="${line#===}"
|
|
GROUP_PATHS=()
|
|
elif [[ "$line" == "---" ]]; then
|
|
flush_p4_group
|
|
elif [[ -n "$line" ]]; then
|
|
GROUP_PATHS+=("$line")
|
|
fi
|
|
done < /tmp/dups-grouped.txt
|
|
flush_p4_group # flush last group
|
|
|
|
P4_GROUPS=$(grep -c '^--- P4 ' "$LOG" 2>/dev/null; true)
|
|
log ""
|
|
log "Pass 4 complete: $P4_GROUPS cross-album dup groups"
|
|
|
|
rm -f /tmp/beets-all-paths.txt /tmp/beets-id-paths.txt /tmp/all-tracks.txt /tmp/dups-grouped.txt
|
|
|
|
TOTAL_GROUPS=$(( P1_GROUPS + P2_GROUPS + P3_GROUPS + P4_GROUPS ))
|
|
KEPT=$(grep -c '^ KEEP ' "$LOG" 2>/dev/null; true)
|
|
DELETED=$(grep -c '^ DELETE ' "$LOG" 2>/dev/null; true)
|
|
|
|
log ""
|
|
log "=== Summary: $TOTAL_GROUPS dup groups ($P1_GROUPS numbered-sibling, $P2_GROUPS case-insensitive, $P3_GROUPS normalized, $P4_GROUPS cross-album), $KEPT kept, $DELETED to delete ==="
|
|
[[ $APPLY -eq 0 ]] && log "DRY RUN — re-run with --apply to actually delete"
|
|
|
|
# Explicit success. Without this the script's exit status is the previous
|
|
# line's, which is non-zero in --apply mode (the `[[ $APPLY -eq 0 ]]` test
|
|
# fails), making a successful apply look like a failed run to the caller.
|
|
# With set -e, any real failure above aborts before reaching here.
|
|
exit 0
|