#!/bin/bash # Walk the beets library for duplicates using three passes: # # Pass 1 — numbered siblings: find every *.N.ext in beets where *.ext also # exists on disk in the same directory; keep the better file. # # Pass 2 — case-insensitive beet-level dedup: pull all tracks from beets, # group by lowercase (albumartist, album, title); any group with # 2+ members is a duplicate set — keep FLAC > MP3 > largest file. # # Pass 3 — normalized beet-level dedup: same as Pass 2 but the key strips # all non-alphanumeric characters before grouping. Catches dupes # whose tags differ only by separator/punctuation/feat. credit # ("Vol. 7" vs ", Vol. 7"; "Artist1; Artist2" vs "Artist1/Artist2"). # # "Best" is always: FLAC > MP3/other, then largest file within the same format. # # DRY RUN by default. Pass --apply to actually delete. # # --json: additionally emit one JSON line per candidate deletion to stdout # (NDJSON), on top of the normal human log -- for dedup_review_service to # parse into dedup_runs/dedup_candidates. Purely additive; the human log # format is unchanged. # --only-paths FILE: in --apply mode, only actually delete a candidate if # its delete_path appears (one per line) in FILE -- everything else in a # group still gets ranked/logged/reported as normal, just not deleted. # This lets the confirm step in the dedup UI re-run the exact same # ranking/safety logic and apply only what a human explicitly approved, # without --only-paths this flag is a no-op and --apply deletes everything # as before (preserves existing behavior for anyone invoking this # directly). set -euo pipefail APPLY=0 JSON_MODE=0 ONLY_PATHS_FILE="" while [[ $# -gt 0 ]]; do case "$1" in --apply) APPLY=1; shift ;; --json) JSON_MODE=1; shift ;; --only-paths) ONLY_PATHS_FILE="$2"; shift 2 ;; *) shift ;; esac done declare -A ONLY_PATHS=() if [[ -n "$ONLY_PATHS_FILE" ]]; then while IFS= read -r p; do [[ -n "$p" ]] && ONLY_PATHS["$p"]=1 done < "$ONLY_PATHS_FILE" fi LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$(date +%Y%m%d-%H%M%S).log mkdir -p "$(dirname "$LOG")" log() { echo "$*" | tee -a "$LOG"; } # JSON lines go to stdout ONLY (never through log()/tee, so they never end # up interleaved with the human-readable $LOG file -- a caller wanting # structured output should capture this process's stdout separately). # Return 0 in non-JSON mode: this is called as a standalone statement, so under # `set -e` a non-zero return (the default dry-run path) would abort the script. json_emit() { [[ $JSON_MODE -eq 1 ]] || return 0; printf '%s\n' "$1"; } json_escape() { local s="$1" s="${s//\\/\\\\}" s="${s//\"/\\\"}" printf '%s' "$s" } if [[ $APPLY -eq 1 ]]; then log "[$(date -Iseconds)] === DEDUP (APPLY) ===" else log "[$(date -Iseconds)] === DEDUP (DRY RUN — pass --apply to delete) ===" fi # Counters: tracked via log grep at the end (avoids bash subshell pitfalls). # Functions write KEEP/DELETE lines with consistent prefixes; summary greps them. # Convert container path (/music/...) to host path (${MUSIC_DATA_DIR:-/data/music}/Library/...) host_path() { echo "${MUSIC_DATA_DIR:-/data/music}/Library${1#/music}"; } SEP=$'\x1c' # ASCII file separator — safe with any music metadata # Rank a file: lower = better. FLAC > MP3 > other (WAV/etc.); clean filename # > sync-conflict artifact (-DESKTOP-, (1), .1.); extended/club mix > radio # edit at same format; then larger wins. # # WAV ranks below MP3 deliberately: WAV has no real tag container and in this # library only appears as a sldl download glitch. Keeping a large WAV over a # tagged MP3 of the same track is never what we want. rank_file() { local hp="$1" local size; size=$(stat -c '%s' "$hp" 2>/dev/null || echo 0) local ext_score=0 [[ "${hp,,}" == *.flac ]] && ext_score=2 [[ "${hp,,}" == *.mp3 ]] && ext_score=1 local dirty_score=0 # (N) copy-artifact check is capped at 2 digits: "(1).flac" is a Windows # copy artifact, but "(2014).flac" is a title ending in a year — marking # those dirty inverted the keep preference toward their .1.flac siblings. if [[ "$hp" =~ -DESKTOP-[A-Z0-9]+ ]] \ || [[ "$hp" =~ \([0-9]{1,2}\)\.[a-z]+$ ]] \ || [[ "$hp" =~ \.[0-9]+\.[a-z]+$ ]]; then dirty_score=1 fi local extended_score=0 shopt -s nocasematch if [[ "$hp" =~ (extended|club\ mix|dj\ edit|dj\ mix|long\ version|original\ mix) ]]; then extended_score=1 fi shopt -u nocasematch echo $((10000000000 - ext_score * 1000000000000 + dirty_score * 100000000000 - extended_score * 50000000000 - size)) } # process_group LABEL ENTRY [ENTRY ...] # ENTRY is either "\x1c" (a track in beets — deleted # via `beet remove id:N`) or a bare host path (a ghost file on disk only; # rm-only to delete). # # Why id queries, not path queries: the beets 2.11 upgrade (~2026-06-19) left # the DB with MIXED path storage — pre-upgrade items store absolute # /music/... paths, post-upgrade imports store library-relative paths. No # single path:/path:: query form matches both populations (this is what made # every dedup DELETE a silent no-op for two weeks). Item ids are storage- # format-proof. process_group() { local label="$1"; shift local ranked="" local hp id entry for entry in "$@"; do [[ -z "$entry" ]] && continue if [[ "$entry" == *"$SEP"* ]]; then id="${entry%%"$SEP"*}" hp=$(host_path "${entry#*"$SEP"}") else id="" hp="$entry" fi [[ -f "$hp" ]] || continue local score; score=$(rank_file "$hp") ranked+="${score}${SEP}${id}${SEP}${hp}"$'\n' done # No files on disk for this group — nothing to do. Return 0 (not 1): callers # invoke process_group as a standalone statement, so under `set -e` a non-zero # return would abort the whole run just because one group's files were gone. [[ -z "$ranked" ]] && return 0 log "" log "--- $label ---" # Pass name for JSON output, matching dedup_candidates.pass_name's # convention (numbered_sibling|case_insensitive|normalized|cross_album_fuzzy). local pass_name="unknown" case "$label" in "P1 numbered-sibling"*) pass_name="numbered_sibling" ;; "P2 case-insensitive"*) pass_name="case_insensitive" ;; "P3 normalized"*) pass_name="normalized" ;; "P4 cross-album"*) pass_name="cross_album_fuzzy" ;; esac local first=1 local keep_path="" while IFS="$SEP" read -r _sk id hp; do [[ -z "$hp" ]] && continue local size_bytes; size_bytes=$(stat -c '%s' "$hp" 2>/dev/null || echo 0) local size_h; size_h=$(numfmt --to=iec --suffix=B "$size_bytes") if [[ $first -eq 1 ]]; then log " KEEP ($size_h) $hp" keep_path="$hp" first=0 else log " DELETE ($size_h) $hp" json_emit "{\"pass\":\"${pass_name}\",\"keep_path\":\"$(json_escape "$keep_path")\",\"delete_path\":\"$(json_escape "$hp")\",\"delete_size_bytes\":${size_bytes}}" if [[ $APPLY -eq 1 ]]; then # With --only-paths given, only delete entries the caller explicitly # confirmed; without it, delete everything (original behavior). if [[ -n "$ONLY_PATHS_FILE" && -z "${ONLY_PATHS[$hp]:-}" ]]; then log " (skipped — not in --only-paths confirm list)" elif [[ -n "$id" ]]; then # In beets — beet remove -d removes from DB + disk. Best-effort: a # single failed removal is logged, not fatal, so the rest of the # confirmed deletions still proceed. beet remove -d -f "id:${id}" >> "$LOG" 2>&1 || log " WARN: beet remove id:${id} failed" else # Ghost file — only on disk; just rm rm -f "$hp" 2>>"$LOG" || log " WARN: rm failed for $hp" fi fi fi done < <(echo "$ranked" | grep -v '^$' | sort -n) return 0 # explicit: the loop's last command status must not leak out under set -e } # ============================================================ # Pass 1: numbered siblings # Find every *.N.ext in beets where *.ext also exists on disk. # ============================================================ log "" log "[$(date -Iseconds)] === Pass 1: numbered siblings ===" # Dump id + container path from beets once. beets normalizes $path for # display (always /music/-prefixed) even though the DB stores a mix of # absolute and relative — so the display paths are safe to compare/convert, # and the ids are what we hand to beet remove/move. beet ls -f "\$id${SEP}\$path" 2>/dev/null > /tmp/beets-id-paths.txt cut -d"$SEP" -f2- /tmp/beets-id-paths.txt > /tmp/beets-all-paths.txt while IFS="$SEP" read -r numbered_id numbered_cp; do [[ -z "$numbered_cp" ]] && continue ext="${numbered_cp##*.}" canonical_cp=$(echo "$numbered_cp" | sed -E "s/\.[0-9]+\.${ext}$/.${ext}/") [[ "$canonical_cp" == "$numbered_cp" ]] && continue # no .N. found host_numbered=$(host_path "$numbered_cp") host_canonical=$(host_path "$canonical_cp") [[ -f "$host_numbered" ]] || continue [[ -f "$host_canonical" ]] || continue # If the canonical is also in beets, Pass 2 handles this pair via beet remove -d. # Skip here to avoid a partial rm that leaves a stale beets DB entry. grep -qxF "$canonical_cp" /tmp/beets-all-paths.txt && continue score_numbered=$(rank_file "$host_numbered") score_canonical=$(rank_file "$host_canonical") if [[ $score_canonical -le $score_numbered ]]; then # Canonical wins: delete numbered (in beets), keep canonical (ghost) process_group "P1 numbered-sibling: ${numbered_cp##/music/}" \ "$host_canonical" "${numbered_id}${SEP}${numbered_cp}" if [[ $APPLY -eq 1 ]]; then # canonical is now a ghost file; import it into beets beet import -q -s "${canonical_cp}" >> "$LOG" 2>&1 || log " WARN: beet import failed for ${canonical_cp}" fi else # Numbered wins (canonical is ghost): rm the ghost, then beet move to rename # the numbered file to the canonical path (id query — see process_group). process_group "P1 numbered-sibling: ${numbered_cp##/music/}" \ "${numbered_id}${SEP}${numbered_cp}" "$host_canonical" if [[ $APPLY -eq 1 ]]; then beet move "id:${numbered_id}" >> "$LOG" 2>&1 || log " WARN: beet move id:${numbered_id} failed" fi fi done < <(grep -E '\.[0-9]+\.(flac|mp3|m4a|ogg|opus|wav)$' /tmp/beets-id-paths.txt) P1_GROUPS=$(grep -c '^--- P1 ' "$LOG" 2>/dev/null; true) log "" log "Pass 1 complete: $P1_GROUPS numbered-sibling groups" # ============================================================ # Pass 2: case-insensitive beet-level dedup # Re-fetch track list so Pass 1 deletions are reflected. # ============================================================ log "" log "[$(date -Iseconds)] === Pass 2: case-insensitive title/album/artist dedup ===" beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \ 2>/dev/null > /tmp/all-tracks.txt gawk -F"${SEP}" -v sep="$SEP" ' # Sentinel titles carry zero identity — multiple distinct tracks can land on # the same value when spotify-retag misses and the Soulseek source had a # blank/placeholder title. Grouping by sentinel title = deleting different # tracks that happened to share a placeholder. Common offenders: # "" fully empty title # "[unknown]" "unknown" sldl/beets fallback # "track 1" .. "track NN" stock CD-rip placeholder function is_sentinel_title(t, lt) { lt = tolower(t) gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt) if (lt == "") return 1 if (lt == "[unknown]" || lt == "unknown") return 1 if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1 return 0 } { if (is_sentinel_title($4)) next key = tolower($2) sep tolower($3) sep tolower($4) paths[key] = paths[key] $1 sep $5 "\n" count[key]++ } END { for (k in paths) { if (count[k] >= 2) print "===" k "\n" paths[k] "---" } }' /tmp/all-tracks.txt > /tmp/dups-grouped.txt GROUP_KEY="" GROUP_PATHS=() flush_p2_group() { [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; } process_group "P2 case-insensitive: $GROUP_KEY" "${GROUP_PATHS[@]}" GROUP_PATHS=() } while IFS= read -r line; do if [[ "$line" == ===* ]]; then flush_p2_group GROUP_KEY="${line#===}" GROUP_PATHS=() elif [[ "$line" == "---" ]]; then flush_p2_group elif [[ -n "$line" ]]; then GROUP_PATHS+=("$line") fi done < /tmp/dups-grouped.txt flush_p2_group # flush last group P2_GROUPS=$(grep -c '^--- P2 ' "$LOG" 2>/dev/null; true) log "" log "Pass 2 complete: $P2_GROUPS case-insensitive dup groups" # ============================================================ # Pass 3: normalized-key dedup (strip non-alphanumerics) # Re-fetch track list so Pass 2 deletions are reflected. # ============================================================ log "" log "[$(date -Iseconds)] === Pass 3: normalized title/album/artist dedup ===" beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \ 2>/dev/null > /tmp/all-tracks.txt # Normalize: lowercase + strip everything except a-z 0-9. Group by the # concatenation. Suppresses dupe pairs already caught by Pass 2 (same exact # key) by also tracking the raw key per group and skipping single-key groups. gawk -F"${SEP}" -v sep="$SEP" ' function norm(s, t) { t = tolower(s) gsub(/[^a-z0-9]/, "", t) return t } function is_sentinel_title(t, lt) { lt = tolower(t) gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt) if (lt == "") return 1 if (lt == "[unknown]" || lt == "unknown") return 1 if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1 return 0 } { if (is_sentinel_title($4)) next raw = tolower($2) sep tolower($3) sep tolower($4) na = norm($2); nb = norm($3); nc = norm($4) # Title is the only field guaranteed unique per track. If it normalizes # to empty (all unicode/punctuation), refuse to group — would cause false # positives across unrelated unicode-titled tracks. A 1-char normalized # title is fine as long as artist+album also normalize non-empty # (covers "ID"/"OP"/single-letter tracks; rejects all-unicode titles # that collapse to ""). if (length(nc) == 0) next if (length(nc) < 3 && (length(na) == 0 || length(nb) == 0)) next k = na sep nb sep nc paths[k] = paths[k] $1 sep $5 "\n" count[k]++ raws[k][raw] = 1 } END { for (k in paths) { if (count[k] < 2) continue # Skip groups that collapse to a single raw key — those were Pass 2 dups. raw_count = 0 for (r in raws[k]) raw_count++ if (raw_count < 2) continue print "===" k "\n" paths[k] "---" } }' /tmp/all-tracks.txt > /tmp/dups-grouped.txt GROUP_KEY="" GROUP_PATHS=() flush_p3_group() { [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; } process_group "P3 normalized: $GROUP_KEY" "${GROUP_PATHS[@]}" GROUP_PATHS=() } while IFS= read -r line; do if [[ "$line" == ===* ]]; then flush_p3_group GROUP_KEY="${line#===}" GROUP_PATHS=() elif [[ "$line" == "---" ]]; then flush_p3_group elif [[ -n "$line" ]]; then GROUP_PATHS+=("$line") fi done < /tmp/dups-grouped.txt flush_p3_group # flush last group P3_GROUPS=$(grep -c '^--- P3 ' "$LOG" 2>/dev/null; true) log "" log "Pass 3 complete: $P3_GROUPS normalized dup groups" # ============================================================ # Pass 4: cross-album dedup (album-agnostic key) # Catches cases where the same track exists once as a single and once on a # compilation/Beatport-style album — the album field always disagrees, so # Pass 2/3 miss them. Keys on (artist_norm, title_base_norm) where title_base # strips trailing mix-version suffixes ("(Extended Mix)", " - Radio Edit", # bare " Extended Mix" with double-space, etc.). # # Allowlist: artists who legitimately have the same track on multiple albums # (live + studio + remix etc.) — these are SKIPPED in Pass 4. Add to # CROSS_ALBUM_KEEP_RE or the album-pattern skip below if needed. # ============================================================ log "" log "[$(date -Iseconds)] === Pass 4: cross-album dedup (artist+title-base) ===" beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \ 2>/dev/null > /tmp/all-tracks.txt # Pass artist + album allowlists into awk via -v. # The artist keep-list (artists whose same-title tracks across albums are NOT # treated as duplicates) is user-editable config, not baked in here. Build the # anchored regex from cross-album-keep.list: normalize each name the same way # the awk norm() does (lowercase, letters+digits only) and OR them together. # An empty/absent list means "protect nobody" -> use a pattern that can never # match a normalized name (which only ever contains [a-z0-9], never '_'). CROSS_ALBUM_KEEP_FILE="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/cross-album-keep.list" if [[ -f "$CROSS_ALBUM_KEEP_FILE" ]]; then # `|| true`: with an all-comments file (the default), the first grep matches # nothing and returns non-zero, which under set -o pipefail + set -e would # otherwise abort here. An empty result is the correct "protect nobody" case. _keep_names=$(grep -vE '^[[:space:]]*(#|$)' "$CROSS_ALBUM_KEEP_FILE" \ | tr 'A-Z' 'a-z' | sed -E 's/[^a-z0-9]//g' | grep -v '^$' | paste -sd'|' - || true) fi if [[ -n "${_keep_names:-}" ]]; then CROSS_ALBUM_KEEP_RE="^(${_keep_names})\$" else CROSS_ALBUM_KEEP_RE='_never_' fi # Album-name patterns that indicate "intentional alt version" — these rows are # filtered out before grouping, so any group needing 2+ matches will only form # from items whose albums DON'T look like remix/live/single-version releases. # Tagging convention: when the user has multiple legitimate versions of a # track, the alt-version copies live on albums with explicit version markers # in the album title ("(Extended Mix) - Single", "(Remixed)", "Live at X"). # Without this filter, Pass 4 would try to dedupe Qrion's "Proud" off the # "I Hope It Lasts Forever (Remixed)" album, Fred again._ studio tracks # against "Apple Music Live" recordings, etc. ALBUM_KEEP_RE='(soundtrack|originalmotionpicture|ost|vgm|gameost|liveat|livein|livefrom|livesession|applemusiclive|applelive|bbclive|bbcsession|mtvunplugged|unplugged|concert|inconcert|extended|remix|remixed|remixes|originalmix|radioedit|radiomix|clubmix|djedit|djmix|dubmix|vocalmix|instrumentalmix|longversion|shortversion|albumversion|originalversion)' gawk -F"${SEP}" -v sep="$SEP" \ -v artist_keep="$CROSS_ALBUM_KEEP_RE" \ -v album_keep="$ALBUM_KEEP_RE" ' function norm(s, t) { t = tolower(s) gsub(/[^a-z0-9]/, "", t) return t } # Strip trailing mix-version suffix from title so "Foo" and "Foo (Extended Mix)" # share the same base. Only strips generic version markers — does NOT strip # remixer-named suffixes like "(Joris Voorn Remix)", which are different songs. function is_sentinel_title(t, lt) { lt = tolower(t) gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt) if (lt == "") return 1 if (lt == "[unknown]" || lt == "unknown") return 1 if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1 return 0 } function strip_mix_suffix(s, t, sfx) { t = tolower(s) sfx = "(extended[[:space:]]*mix|extended[[:space:]]*version|extended|" \ "radio[[:space:]]*edit|radio[[:space:]]*mix|radio[[:space:]]*version|" \ "original[[:space:]]*mix|original[[:space:]]*version|" \ "club[[:space:]]*mix|dj[[:space:]]*edit|dj[[:space:]]*mix|" \ "long[[:space:]]*version|short[[:space:]]*version|" \ "single[[:space:]]*version|album[[:space:]]*version)" # parenthetical: " (Extended Mix)" t = gensub("[[:space:]]*\\(" sfx "\\)[[:space:]]*$", "", "g", t) # dash-separated: " - Extended Mix" t = gensub("[[:space:]]*-[[:space:]]+" sfx "[[:space:]]*$", "", "g", t) # bare trailing: " Extended Mix" t = gensub("[[:space:]]+" sfx "[[:space:]]*$", "", "g", t) return t } { id = $1; artist = $2; album = $3; title = $4; path = $5 if (is_sentinel_title(title)) next na = norm(artist) if (na ~ artist_keep) next if (norm(album) ~ album_keep) next tb = strip_mix_suffix(title) ntb = norm(tb) if (length(ntb) < 4) next k = na sep ntb paths[k] = paths[k] id sep path "\n" count[k]++ raws3[k][na sep norm(album) sep norm(title)] = 1 albums[k][album] = 1 } END { for (k in paths) { if (count[k] < 2) continue # Skip if all items share the same Pass 3 normalized key (Pass 3 already handled). rcount = 0; for (r in raws3[k]) rcount++ if (rcount < 2) continue # Require ≥2 distinct album names — otherwise Pass 3 would have grouped them. acount = 0; for (a in albums[k]) acount++ if (acount < 2) continue print "===" k "\n" paths[k] "---" } }' /tmp/all-tracks.txt > /tmp/dups-grouped.txt GROUP_KEY="" GROUP_PATHS=() flush_p4_group() { [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; } process_group "P4 cross-album: $GROUP_KEY" "${GROUP_PATHS[@]}" GROUP_PATHS=() } while IFS= read -r line; do if [[ "$line" == ===* ]]; then flush_p4_group GROUP_KEY="${line#===}" GROUP_PATHS=() elif [[ "$line" == "---" ]]; then flush_p4_group elif [[ -n "$line" ]]; then GROUP_PATHS+=("$line") fi done < /tmp/dups-grouped.txt flush_p4_group # flush last group P4_GROUPS=$(grep -c '^--- P4 ' "$LOG" 2>/dev/null; true) log "" log "Pass 4 complete: $P4_GROUPS cross-album dup groups" rm -f /tmp/beets-all-paths.txt /tmp/beets-id-paths.txt /tmp/all-tracks.txt /tmp/dups-grouped.txt TOTAL_GROUPS=$(( P1_GROUPS + P2_GROUPS + P3_GROUPS + P4_GROUPS )) KEPT=$(grep -c '^ KEEP ' "$LOG" 2>/dev/null; true) DELETED=$(grep -c '^ DELETE ' "$LOG" 2>/dev/null; true) log "" log "=== Summary: $TOTAL_GROUPS dup groups ($P1_GROUPS numbered-sibling, $P2_GROUPS case-insensitive, $P3_GROUPS normalized, $P4_GROUPS cross-album), $KEPT kept, $DELETED to delete ===" [[ $APPLY -eq 0 ]] && log "DRY RUN — re-run with --apply to actually delete" # Explicit success. Without this the script's exit status is the previous # line's, which is non-zero in --apply mode (the `[[ $APPLY -eq 0 ]]` test # fails), making a successful apply look like a failed run to the caller. # With set -e, any real failure above aborts before reaching here. exit 0