alembic/pipeline/lib/dedup-library.sh

#!/bin/bash
# Walk the beets library for duplicates using three passes:
#
#   Pass 1 — numbered siblings: find every *.N.ext in beets where *.ext also
#             exists on disk in the same directory; keep the better file.
#
#   Pass 2 — case-insensitive beet-level dedup: pull all tracks from beets,
#             group by lowercase (albumartist, album, title); any group with
#             2+ members is a duplicate set — keep FLAC > MP3 > largest file.
#
#   Pass 3 — normalized beet-level dedup: same as Pass 2 but the key strips
#             all non-alphanumeric characters before grouping. Catches dupes
#             whose tags differ only by separator/punctuation/feat. credit
#             ("Vol. 7" vs ", Vol. 7"; "Artist1; Artist2" vs "Artist1/Artist2").
#
# "Best" is always: FLAC > MP3/other, then largest file within the same format.
#
# DRY RUN by default. Pass --apply to actually delete.
#
# --json: additionally emit one JSON line per candidate deletion to stdout
#   (NDJSON), on top of the normal human log -- for dedup_review_service to
#   parse into dedup_runs/dedup_candidates. Purely additive; the human log
#   format is unchanged.
# --only-paths FILE: in --apply mode, only actually delete a candidate if
#   its delete_path appears (one per line) in FILE -- everything else in a
#   group still gets ranked/logged/reported as normal, just not deleted.
#   This lets the confirm step in the dedup UI re-run the exact same
#   ranking/safety logic and apply only what a human explicitly approved,
#   without --only-paths this flag is a no-op and --apply deletes everything
#   as before (preserves existing behavior for anyone invoking this
#   directly).

set -u
APPLY=0
JSON_MODE=0
ONLY_PATHS_FILE=""
while [[ $# -gt 0 ]]; do
  case "$1" in
    --apply) APPLY=1; shift ;;
    --json) JSON_MODE=1; shift ;;
    --only-paths) ONLY_PATHS_FILE="$2"; shift 2 ;;
    *) shift ;;
  esac
done

declare -A ONLY_PATHS=()
if [[ -n "$ONLY_PATHS_FILE" ]]; then
  while IFS= read -r p; do
    [[ -n "$p" ]] && ONLY_PATHS["$p"]=1
  done < "$ONLY_PATHS_FILE"
fi

LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$(date +%Y%m%d-%H%M%S).log
mkdir -p "$(dirname "$LOG")"

log() { echo "$*" | tee -a "$LOG"; }

# JSON lines go to stdout ONLY (never through log()/tee, so they never end
# up interleaved with the human-readable $LOG file -- a caller wanting
# structured output should capture this process's stdout separately).
json_emit() { [[ $JSON_MODE -eq 1 ]] && printf '%s\n' "$1"; }

json_escape() {
  local s="$1"
  s="${s//\\/\\\\}"
  s="${s//\"/\\\"}"
  printf '%s' "$s"
}

if [[ $APPLY -eq 1 ]]; then
  log "[$(date -Iseconds)] === DEDUP (APPLY) ==="
else
  log "[$(date -Iseconds)] === DEDUP (DRY RUN — pass --apply to delete) ==="
fi

# Counters: tracked via log grep at the end (avoids bash subshell pitfalls).
# Functions write KEEP/DELETE lines with consistent prefixes; summary greps them.

# Convert container path (/music/...) to host path (${MUSIC_DATA_DIR:-/data/music}/Library/...)
host_path() { echo "${MUSIC_DATA_DIR:-/data/music}/Library${1#/music}"; }

SEP=$'\x1c'  # ASCII file separator — safe with any music metadata

# Rank a file: lower = better. FLAC > MP3 > other (WAV/etc.); clean filename
# > sync-conflict artifact (-DESKTOP-, (1), .1.); extended/club mix > radio
# edit at same format; then larger wins.
#
# WAV ranks below MP3 deliberately: WAV has no real tag container and in this
# library only appears as a sldl download glitch. Keeping a large WAV over a
# tagged MP3 of the same track is never what we want.
rank_file() {
  local hp="$1"
  local size; size=$(stat -c '%s' "$hp" 2>/dev/null || echo 0)
  local ext_score=0
  [[ "${hp,,}" == *.flac ]] && ext_score=2
  [[ "${hp,,}" == *.mp3 ]]  && ext_score=1
  local dirty_score=0
  # (N) copy-artifact check is capped at 2 digits: "(1).flac" is a Windows
  # copy artifact, but "(2014).flac" is a title ending in a year — marking
  # those dirty inverted the keep preference toward their .1.flac siblings.
  if [[ "$hp" =~ -DESKTOP-[A-Z0-9]+ ]] \
     || [[ "$hp" =~ \([0-9]{1,2}\)\.[a-z]+$ ]] \
     || [[ "$hp" =~ \.[0-9]+\.[a-z]+$ ]]; then
    dirty_score=1
  fi
  local extended_score=0
  shopt -s nocasematch
  if [[ "$hp" =~ (extended|club\ mix|dj\ edit|dj\ mix|long\ version|original\ mix) ]]; then
    extended_score=1
  fi
  shopt -u nocasematch
  echo $((10000000000 - ext_score * 1000000000000 + dirty_score * 100000000000 - extended_score * 50000000000 - size))
}

# process_group LABEL ENTRY [ENTRY ...]
# ENTRY is either "<beets-id>\x1c<container-path>" (a track in beets — deleted
# via `beet remove id:N`) or a bare host path (a ghost file on disk only;
# rm-only to delete).
#
# Why id queries, not path queries: the beets 2.11 upgrade (~2026-06-19) left
# the DB with MIXED path storage — pre-upgrade items store absolute
# /music/... paths, post-upgrade imports store library-relative paths. No
# single path:/path:: query form matches both populations (this is what made
# every dedup DELETE a silent no-op for two weeks). Item ids are storage-
# format-proof.
process_group() {
  local label="$1"; shift

  local ranked=""
  local hp id entry
  for entry in "$@"; do
    [[ -z "$entry" ]] && continue
    if [[ "$entry" == *"$SEP"* ]]; then
      id="${entry%%"$SEP"*}"
      hp=$(host_path "${entry#*"$SEP"}")
    else
      id=""
      hp="$entry"
    fi
    [[ -f "$hp" ]] || continue
    local score; score=$(rank_file "$hp")
    ranked+="${score}${SEP}${id}${SEP}${hp}"$'\n'
  done

  [[ -z "$ranked" ]] && return 1  # no files on disk — skip

  log ""
  log "--- $label ---"

  # Pass name for JSON output, matching dedup_candidates.pass_name's
  # convention (numbered_sibling|case_insensitive|normalized|cross_album_fuzzy).
  local pass_name="unknown"
  case "$label" in
    "P1 numbered-sibling"*) pass_name="numbered_sibling" ;;
    "P2 case-insensitive"*) pass_name="case_insensitive" ;;
    "P3 normalized"*)       pass_name="normalized" ;;
    "P4 cross-album"*)      pass_name="cross_album_fuzzy" ;;
  esac

  local first=1
  local keep_path=""
  while IFS="$SEP" read -r _sk id hp; do
    [[ -z "$hp" ]] && continue
    local size_bytes; size_bytes=$(stat -c '%s' "$hp" 2>/dev/null || echo 0)
    local size_h; size_h=$(numfmt --to=iec --suffix=B "$size_bytes")
    if [[ $first -eq 1 ]]; then
      log "  KEEP   ($size_h) $hp"
      keep_path="$hp"
      first=0
    else
      log "  DELETE ($size_h) $hp"
      json_emit "{\"pass\":\"${pass_name}\",\"keep_path\":\"$(json_escape "$keep_path")\",\"delete_path\":\"$(json_escape "$hp")\",\"delete_size_bytes\":${size_bytes}}"
      if [[ $APPLY -eq 1 ]]; then
        # With --only-paths given, only delete entries the caller explicitly
        # confirmed; without it, delete everything (original behavior).
        if [[ -n "$ONLY_PATHS_FILE" && -z "${ONLY_PATHS[$hp]:-}" ]]; then
          log "    (skipped — not in --only-paths confirm list)"
        elif [[ -n "$id" ]]; then
          # In beets — beet remove -d removes from DB + disk.
          beet remove -d -f "id:${id}" >> "$LOG" 2>&1
        else
          # Ghost file — only on disk; just rm
          rm -f "$hp" 2>>"$LOG"
        fi
      fi
    fi
  done < <(echo "$ranked" | grep -v '^$' | sort -n)
}

# ============================================================
# Pass 1: numbered siblings
# Find every *.N.ext in beets where *.ext also exists on disk.
# ============================================================
log ""
log "[$(date -Iseconds)] === Pass 1: numbered siblings ==="

# Dump id + container path from beets once. beets normalizes $path for
# display (always /music/-prefixed) even though the DB stores a mix of
# absolute and relative — so the display paths are safe to compare/convert,
# and the ids are what we hand to beet remove/move.
beet ls -f "\$id${SEP}\$path" 2>/dev/null > /tmp/beets-id-paths.txt
cut -d"$SEP" -f2- /tmp/beets-id-paths.txt > /tmp/beets-all-paths.txt

while IFS="$SEP" read -r numbered_id numbered_cp; do
  [[ -z "$numbered_cp" ]] && continue

  ext="${numbered_cp##*.}"
  canonical_cp=$(echo "$numbered_cp" | sed -E "s/\.[0-9]+\.${ext}$/.${ext}/")
  [[ "$canonical_cp" == "$numbered_cp" ]] && continue  # no .N. found

  host_numbered=$(host_path "$numbered_cp")
  host_canonical=$(host_path "$canonical_cp")

  [[ -f "$host_numbered" ]] || continue
  [[ -f "$host_canonical" ]] || continue

  # If the canonical is also in beets, Pass 2 handles this pair via beet remove -d.
  # Skip here to avoid a partial rm that leaves a stale beets DB entry.
  grep -qxF "$canonical_cp" /tmp/beets-all-paths.txt && continue

  score_numbered=$(rank_file "$host_numbered")
  score_canonical=$(rank_file "$host_canonical")

  if [[ $score_canonical -le $score_numbered ]]; then
    # Canonical wins: delete numbered (in beets), keep canonical (ghost)
    process_group "P1 numbered-sibling: ${numbered_cp##/music/}" \
      "$host_canonical" "${numbered_id}${SEP}${numbered_cp}"
    if [[ $APPLY -eq 1 ]]; then
      # canonical is now a ghost file; import it into beets
      beet import -q -s "${canonical_cp}" >> "$LOG" 2>&1
    fi
  else
    # Numbered wins (canonical is ghost): rm the ghost, then beet move to rename
    # the numbered file to the canonical path (id query — see process_group).
    process_group "P1 numbered-sibling: ${numbered_cp##/music/}" \
      "${numbered_id}${SEP}${numbered_cp}" "$host_canonical"
    if [[ $APPLY -eq 1 ]]; then
      beet move "id:${numbered_id}" >> "$LOG" 2>&1
    fi
  fi
done < <(grep -E '\.[0-9]+\.(flac|mp3|m4a|ogg|opus|wav)$' /tmp/beets-id-paths.txt)

P1_GROUPS=$(grep -c '^--- P1 ' "$LOG" 2>/dev/null; true)
log ""
log "Pass 1 complete: $P1_GROUPS numbered-sibling groups"

# ============================================================
# Pass 2: case-insensitive beet-level dedup
# Re-fetch track list so Pass 1 deletions are reflected.
# ============================================================
log ""
log "[$(date -Iseconds)] === Pass 2: case-insensitive title/album/artist dedup ==="

beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
  2>/dev/null > /tmp/all-tracks.txt

gawk -F"${SEP}" -v sep="$SEP" '
# Sentinel titles carry zero identity — multiple distinct tracks can land on
# the same value when spotify-retag misses and the Soulseek source had a
# blank/placeholder title. Grouping by sentinel title = deleting different
# tracks that happened to share a placeholder. Common offenders:
#   ""                       fully empty title
#   "[unknown]" "unknown"    sldl/beets fallback
#   "track 1" .. "track NN"  stock CD-rip placeholder
function is_sentinel_title(t,    lt) {
  lt = tolower(t)
  gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
  if (lt == "") return 1
  if (lt == "[unknown]" || lt == "unknown") return 1
  if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
  return 0
}
{
  if (is_sentinel_title($4)) next
  key = tolower($2) sep tolower($3) sep tolower($4)
  paths[key] = paths[key] $1 sep $5 "\n"
  count[key]++
}
END {
  for (k in paths) {
    if (count[k] >= 2) print "===" k "\n" paths[k] "---"
  }
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt

GROUP_KEY=""
GROUP_PATHS=()

flush_p2_group() {
  [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
  process_group "P2 case-insensitive: $GROUP_KEY" "${GROUP_PATHS[@]}"
  GROUP_PATHS=()
}

while IFS= read -r line; do
  if [[ "$line" == ===* ]]; then
    flush_p2_group
    GROUP_KEY="${line#===}"
    GROUP_PATHS=()
  elif [[ "$line" == "---" ]]; then
    flush_p2_group
  elif [[ -n "$line" ]]; then
    GROUP_PATHS+=("$line")
  fi
done < /tmp/dups-grouped.txt
flush_p2_group  # flush last group

P2_GROUPS=$(grep -c '^--- P2 ' "$LOG" 2>/dev/null; true)
log ""
log "Pass 2 complete: $P2_GROUPS case-insensitive dup groups"

# ============================================================
# Pass 3: normalized-key dedup (strip non-alphanumerics)
# Re-fetch track list so Pass 2 deletions are reflected.
# ============================================================
log ""
log "[$(date -Iseconds)] === Pass 3: normalized title/album/artist dedup ==="

beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
  2>/dev/null > /tmp/all-tracks.txt

# Normalize: lowercase + strip everything except a-z 0-9. Group by the
# concatenation. Suppresses dupe pairs already caught by Pass 2 (same exact
# key) by also tracking the raw key per group and skipping single-key groups.
gawk -F"${SEP}" -v sep="$SEP" '
function norm(s,    t) {
  t = tolower(s)
  gsub(/[^a-z0-9]/, "", t)
  return t
}
function is_sentinel_title(t,    lt) {
  lt = tolower(t)
  gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
  if (lt == "") return 1
  if (lt == "[unknown]" || lt == "unknown") return 1
  if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
  return 0
}
{
  if (is_sentinel_title($4)) next
  raw = tolower($2) sep tolower($3) sep tolower($4)
  na = norm($2); nb = norm($3); nc = norm($4)
  # Title is the only field guaranteed unique per track. If it normalizes
  # to empty (all unicode/punctuation), refuse to group — would cause false
  # positives across unrelated unicode-titled tracks. A 1-char normalized
  # title is fine as long as artist+album also normalize non-empty
  # (covers "ID"/"OP"/single-letter tracks; rejects all-unicode titles
  # that collapse to "").
  if (length(nc) == 0) next
  if (length(nc) < 3 && (length(na) == 0 || length(nb) == 0)) next
  k = na sep nb sep nc
  paths[k] = paths[k] $1 sep $5 "\n"
  count[k]++
  raws[k][raw] = 1
}
END {
  for (k in paths) {
    if (count[k] < 2) continue
    # Skip groups that collapse to a single raw key — those were Pass 2 dups.
    raw_count = 0
    for (r in raws[k]) raw_count++
    if (raw_count < 2) continue
    print "===" k "\n" paths[k] "---"
  }
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt

GROUP_KEY=""
GROUP_PATHS=()

flush_p3_group() {
  [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
  process_group "P3 normalized: $GROUP_KEY" "${GROUP_PATHS[@]}"
  GROUP_PATHS=()
}

while IFS= read -r line; do
  if [[ "$line" == ===* ]]; then
    flush_p3_group
    GROUP_KEY="${line#===}"
    GROUP_PATHS=()
  elif [[ "$line" == "---" ]]; then
    flush_p3_group
  elif [[ -n "$line" ]]; then
    GROUP_PATHS+=("$line")
  fi
done < /tmp/dups-grouped.txt
flush_p3_group  # flush last group

P3_GROUPS=$(grep -c '^--- P3 ' "$LOG" 2>/dev/null; true)
log ""
log "Pass 3 complete: $P3_GROUPS normalized dup groups"

# ============================================================
# Pass 4: cross-album dedup (album-agnostic key)
# Catches cases where the same track exists once as a single and once on a
# compilation/Beatport-style album — the album field always disagrees, so
# Pass 2/3 miss them. Keys on (artist_norm, title_base_norm) where title_base
# strips trailing mix-version suffixes ("(Extended Mix)", " - Radio Edit",
# bare "  Extended Mix" with double-space, etc.).
#
# Allowlist: artists who legitimately have the same track on multiple albums
# (live + studio + remix etc.) — these are SKIPPED in Pass 4. Add to
# CROSS_ALBUM_KEEP_RE or the album-pattern skip below if needed.
# ============================================================
log ""
log "[$(date -Iseconds)] === Pass 4: cross-album dedup (artist+title-base) ==="

beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
  2>/dev/null > /tmp/all-tracks.txt

# Pass artist + album allowlists into awk via -v.
# The artist keep-list (artists whose same-title tracks across albums are NOT
# treated as duplicates) is user-editable config, not baked in here. Build the
# anchored regex from cross-album-keep.list: normalize each name the same way
# the awk norm() does (lowercase, letters+digits only) and OR them together.
# An empty/absent list means "protect nobody" -> use a pattern that can never
# match a normalized name (which only ever contains [a-z0-9], never '_').
CROSS_ALBUM_KEEP_FILE="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/cross-album-keep.list"
if [[ -f "$CROSS_ALBUM_KEEP_FILE" ]]; then
  _keep_names=$(grep -vE '^[[:space:]]*(#|$)' "$CROSS_ALBUM_KEEP_FILE" \
    | tr 'A-Z' 'a-z' | sed -E 's/[^a-z0-9]//g' | grep -v '^$' | paste -sd'|' -)
fi
if [[ -n "${_keep_names:-}" ]]; then
  CROSS_ALBUM_KEEP_RE="^(${_keep_names})\$"
else
  CROSS_ALBUM_KEEP_RE='_never_'
fi
# Album-name patterns that indicate "intentional alt version" — these rows are
# filtered out before grouping, so any group needing 2+ matches will only form
# from items whose albums DON'T look like remix/live/single-version releases.
# Tagging convention: when the user has multiple legitimate versions of a
# track, the alt-version copies live on albums with explicit version markers
# in the album title ("(Extended Mix) - Single", "(Remixed)", "Live at X").
# Without this filter, Pass 4 would try to dedupe Qrion's "Proud" off the
# "I Hope It Lasts Forever (Remixed)" album, Fred again._ studio tracks
# against "Apple Music Live" recordings, etc.
ALBUM_KEEP_RE='(soundtrack|originalmotionpicture|ost|vgm|gameost|liveat|livein|livefrom|livesession|applemusiclive|applelive|bbclive|bbcsession|mtvunplugged|unplugged|concert|inconcert|extended|remix|remixed|remixes|originalmix|radioedit|radiomix|clubmix|djedit|djmix|dubmix|vocalmix|instrumentalmix|longversion|shortversion|albumversion|originalversion)'

gawk -F"${SEP}" -v sep="$SEP" \
    -v artist_keep="$CROSS_ALBUM_KEEP_RE" \
    -v album_keep="$ALBUM_KEEP_RE" '
function norm(s,    t) {
  t = tolower(s)
  gsub(/[^a-z0-9]/, "", t)
  return t
}
# Strip trailing mix-version suffix from title so "Foo" and "Foo (Extended Mix)"
# share the same base. Only strips generic version markers — does NOT strip
# remixer-named suffixes like "(Joris Voorn Remix)", which are different songs.
function is_sentinel_title(t,    lt) {
  lt = tolower(t)
  gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
  if (lt == "") return 1
  if (lt == "[unknown]" || lt == "unknown") return 1
  if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
  return 0
}
function strip_mix_suffix(s,    t, sfx) {
  t = tolower(s)
  sfx = "(extended[[:space:]]*mix|extended[[:space:]]*version|extended|" \
        "radio[[:space:]]*edit|radio[[:space:]]*mix|radio[[:space:]]*version|" \
        "original[[:space:]]*mix|original[[:space:]]*version|" \
        "club[[:space:]]*mix|dj[[:space:]]*edit|dj[[:space:]]*mix|" \
        "long[[:space:]]*version|short[[:space:]]*version|" \
        "single[[:space:]]*version|album[[:space:]]*version)"
  # parenthetical: " (Extended Mix)"
  t = gensub("[[:space:]]*\\(" sfx "\\)[[:space:]]*$", "", "g", t)
  # dash-separated: " - Extended Mix"
  t = gensub("[[:space:]]*-[[:space:]]+" sfx "[[:space:]]*$", "", "g", t)
  # bare trailing: "  Extended Mix"
  t = gensub("[[:space:]]+" sfx "[[:space:]]*$", "", "g", t)
  return t
}
{
  id = $1; artist = $2; album = $3; title = $4; path = $5
  if (is_sentinel_title(title)) next
  na = norm(artist)
  if (na ~ artist_keep) next
  if (norm(album) ~ album_keep) next
  tb = strip_mix_suffix(title)
  ntb = norm(tb)
  if (length(ntb) < 4) next
  k = na sep ntb
  paths[k] = paths[k] id sep path "\n"
  count[k]++
  raws3[k][na sep norm(album) sep norm(title)] = 1
  albums[k][album] = 1
}
END {
  for (k in paths) {
    if (count[k] < 2) continue
    # Skip if all items share the same Pass 3 normalized key (Pass 3 already handled).
    rcount = 0; for (r in raws3[k]) rcount++
    if (rcount < 2) continue
    # Require ≥2 distinct album names — otherwise Pass 3 would have grouped them.
    acount = 0; for (a in albums[k]) acount++
    if (acount < 2) continue
    print "===" k "\n" paths[k] "---"
  }
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt

GROUP_KEY=""
GROUP_PATHS=()

flush_p4_group() {
  [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
  process_group "P4 cross-album: $GROUP_KEY" "${GROUP_PATHS[@]}"
  GROUP_PATHS=()
}

while IFS= read -r line; do
  if [[ "$line" == ===* ]]; then
    flush_p4_group
    GROUP_KEY="${line#===}"
    GROUP_PATHS=()
  elif [[ "$line" == "---" ]]; then
    flush_p4_group
  elif [[ -n "$line" ]]; then
    GROUP_PATHS+=("$line")
  fi
done < /tmp/dups-grouped.txt
flush_p4_group  # flush last group

P4_GROUPS=$(grep -c '^--- P4 ' "$LOG" 2>/dev/null; true)
log ""
log "Pass 4 complete: $P4_GROUPS cross-album dup groups"

rm -f /tmp/beets-all-paths.txt /tmp/beets-id-paths.txt /tmp/all-tracks.txt /tmp/dups-grouped.txt

TOTAL_GROUPS=$(( P1_GROUPS + P2_GROUPS + P3_GROUPS + P4_GROUPS ))
KEPT=$(grep -c '^  KEEP   ' "$LOG" 2>/dev/null; true)
DELETED=$(grep -c '^  DELETE ' "$LOG" 2>/dev/null; true)

log ""
log "=== Summary: $TOTAL_GROUPS dup groups ($P1_GROUPS numbered-sibling, $P2_GROUPS case-insensitive, $P3_GROUPS normalized, $P4_GROUPS cross-album), $KEPT kept, $DELETED to delete ==="
[[ $APPLY -eq 0 ]] && log "DRY RUN — re-run with --apply to actually delete"