#!/bin/bash
# Walk the beets library for duplicates using three passes:
#
#   Pass 1 — numbered siblings: find every *.N.ext in beets where *.ext also
#             exists on disk in the same directory; keep the better file.
#
#   Pass 2 — case-insensitive beet-level dedup: pull all tracks from beets,
#             group by lowercase (albumartist, album, title); any group with
#             2+ members is a duplicate set — keep FLAC > MP3 > largest file.
#
#   Pass 3 — normalized beet-level dedup: same as Pass 2 but the key strips
#             all non-alphanumeric characters before grouping. Catches dupes
#             whose tags differ only by separator/punctuation/feat. credit
#             ("Vol. 7" vs ", Vol. 7"; "Artist1; Artist2" vs "Artist1/Artist2").
#
# "Best" is always: FLAC > MP3/other, then largest file within the same format.
#
# DRY RUN by default. Pass --apply to actually delete.
#
# --json: additionally emit one JSON line per candidate deletion to stdout
#   (NDJSON), on top of the normal human log -- for dedup_review_service to
#   parse into dedup_runs/dedup_candidates. Purely additive; the human log
#   format is unchanged.
# --only-paths FILE: in --apply mode, only actually delete a candidate if
#   its delete_path appears (one per line) in FILE -- everything else in a
#   group still gets ranked/logged/reported as normal, just not deleted.
#   This lets the confirm step in the dedup UI re-run the exact same
#   ranking/safety logic and apply only what a human explicitly approved,
#   without --only-paths this flag is a no-op and --apply deletes everything
#   as before (preserves existing behavior for anyone invoking this
#   directly).

set -euo pipefail
APPLY=0
JSON_MODE=0
ONLY_PATHS_FILE=""
AUTO_APPLY_NUMBERED=0
while [[ $# -gt 0 ]]; do
  case "$1" in
    --apply) APPLY=1; shift ;;
    --json) JSON_MODE=1; shift ;;
    --only-paths) ONLY_PATHS_FILE="$2"; shift 2 ;;
    # Delete two kinds of match unconditionally, independent of
    # --apply/--only-paths, while everything else stays dry-run/log-only:
    #   - numbered-sibling twins (see is_numbered_twin): same dir, same name,
    #     same extension, just the ".N" collision suffix -- the same download
    #     landing twice.
    #   - format upgrades within an exact tag match (see is_format_upgrade):
    #     Pass 2/3 already proved same song via identical tags; a different
    #     extension there just means "better format vs. worse", never a
    #     different version.
    # Excludes Pass 4 (cross-album fuzzy) entirely -- matching across
    # different albums/directories is exactly where a different master or
    # DJ-mix edit can share tags without being interchangeable, so it always
    # needs a human. See dedup_review_service.scan().
    --auto-apply-numbered) AUTO_APPLY_NUMBERED=1; shift ;;
    *) shift ;;
  esac
done

declare -A ONLY_PATHS=()
if [[ -n "$ONLY_PATHS_FILE" ]]; then
  while IFS= read -r p; do
    [[ -n "$p" ]] && ONLY_PATHS["$p"]=1
  done < "$ONLY_PATHS_FILE"
fi

LOG=${ALEMBIC_CONFIG_DIR:-/config}/logs/dedup-$(date +%Y%m%d-%H%M%S).log
mkdir -p "$(dirname "$LOG")"

log() { echo "$*" | tee -a "$LOG"; }

# JSON lines go to stdout ONLY (never through log()/tee, so they never end
# up interleaved with the human-readable $LOG file -- a caller wanting
# structured output should capture this process's stdout separately).
# Return 0 in non-JSON mode: this is called as a standalone statement, so under
# `set -e` a non-zero return (the default dry-run path) would abort the script.
json_emit() { [[ $JSON_MODE -eq 1 ]] || return 0; printf '%s\n' "$1"; }

json_escape() {
  local s="$1"
  s="${s//\\/\\\\}"
  s="${s//\"/\\\"}"
  printf '%s' "$s"
}

if [[ $APPLY -eq 1 ]]; then
  log "[$(date -Iseconds)] === DEDUP (APPLY) ==="
else
  log "[$(date -Iseconds)] === DEDUP (DRY RUN — pass --apply to delete) ==="
fi

# Counters: tracked via log grep at the end (avoids bash subshell pitfalls).
# Functions write KEEP/DELETE lines with consistent prefixes; summary greps them.

# Resolve a beets-displayed path to a real path in this container. Since the
# 2026-07-08 path rewrite, beets displays ${MUSIC_DATA_DIR:-/data/music}/Library/...
# paths that are directly usable — pass those through UNCHANGED. Only legacy
# /music/... display paths (the pre-rewrite transitional mount) still need the
# prefix swap. Blindly prepending the library dir to an already-correct path
# (the old behavior) produced /data/music/Library/data/music/Library/... —
# nonexistent, so every candidate failed the -f check and every pass reported
# 0 groups from 2026-07-08 until this fix.
host_path() {
  local p="$1"
  if [[ "$p" == /music/* ]]; then
    echo "${MUSIC_DATA_DIR:-/data/music}/Library${p#/music}"
  else
    echo "$p"
  fi
}

SEP=$'\x1c'  # ASCII file separator — safe with any music metadata

# True if two paths are the SAME directory, SAME extension, and identical
# basenames except for a numeric ".N" infix sldl/beets inserts on a filename
# collision (e.g. "Title.flac" vs "Title.1.flac"). This is deliberately
# narrower than any tag-based pass: no fuzzy matching, no cross-album
# ambiguity, just the literal same download landing twice. Used to
# auto-delete regardless of which pass found the pair -- Pass 1 only fires
# when the canonical name is a beets ghost; the very common case where BOTH
# copies are already tracked in beets (so Pass 1 defers) surfaces instead via
# Pass 2/3's tag-based grouping, and still deserves the same free pass.
is_numbered_twin() {
  local a="$1" b="$2"
  [[ "${a%/*}" == "${b%/*}" ]] || return 1
  local ba="${a##*/}" bb="${b##*/}"
  local ext="${ba##*.}"
  [[ "${ext,,}" == "${bb##*.}" ]] || return 1
  [[ "$(echo "$ba" | sed -E "s/\.[0-9]+\.${ext}\$/.${ext}/")" \
     == "$(echo "$bb" | sed -E "s/\.[0-9]+\.${ext}\$/.${ext}/")" ]]
}

# True if this pair is a same-song format upgrade found by an EXACT tag match
# (Pass 2 case-insensitive or Pass 3 normalized -- both require identical
# albumartist+album+title after case/punctuation folding, so "same song" is
# already proven by the pass itself) where the two files simply have
# different extensions. rank_file() always ranks FLAC ahead of any other
# format regardless of size/dirty-name/etc, so within a tag-exact group a
# cross-extension pair unconditionally means "the format-preferred copy vs.
# a lesser one" -- no judgment call. Deliberately excludes Pass 4
# (cross_album_fuzzy): that pass matches by artist+title-base ACROSS
# different albums/directories, which is exactly where a different master,
# DJ-mix edit, or reissue can share tags without being an interchangeable
# copy -- those still need a human to look at album/context before deleting
# either side.
is_format_upgrade() {
  local pass="$1" keep="$2" delete="$3"
  [[ "$pass" == "case_insensitive" || "$pass" == "normalized" ]] || return 1
  [[ "${keep##*.}" != "${delete##*.}" ]]
}

# Rank a file: lower = better. FLAC > MP3 > other (WAV/etc.); clean filename
# > sync-conflict artifact (-DESKTOP-, (1), .1.); extended/club mix > radio
# edit at same format; then larger wins.
#
# WAV ranks below MP3 deliberately: WAV has no real tag container and in this
# library only appears as a sldl download glitch. Keeping a large WAV over a
# tagged MP3 of the same track is never what we want.
rank_file() {
  local hp="$1"
  local size; size=$(stat -c '%s' "$hp" 2>/dev/null || echo 0)
  local ext_score=0
  [[ "${hp,,}" == *.flac ]] && ext_score=2
  [[ "${hp,,}" == *.mp3 ]]  && ext_score=1
  local dirty_score=0
  # (N) copy-artifact check is capped at 2 digits: "(1).flac" is a Windows
  # copy artifact, but "(2014).flac" is a title ending in a year — marking
  # those dirty inverted the keep preference toward their .1.flac siblings.
  if [[ "$hp" =~ -DESKTOP-[A-Z0-9]+ ]] \
     || [[ "$hp" =~ \([0-9]{1,2}\)\.[a-z]+$ ]] \
     || [[ "$hp" =~ \.[0-9]+\.[a-z]+$ ]]; then
    dirty_score=1
  fi
  local extended_score=0
  shopt -s nocasematch
  if [[ "$hp" =~ (extended|club\ mix|dj\ edit|dj\ mix|long\ version|original\ mix) ]]; then
    extended_score=1
  fi
  shopt -u nocasematch
  echo $((10000000000 - ext_score * 1000000000000 + dirty_score * 100000000000 - extended_score * 50000000000 - size))
}

# process_group LABEL ENTRY [ENTRY ...]
# ENTRY is either "<beets-id>\x1c<container-path>" (a track in beets — deleted
# via `beet remove id:N`) or a bare host path (a ghost file on disk only;
# rm-only to delete).
#
# Why id queries, not path queries: the beets 2.11 upgrade (~2026-06-19) left
# the DB with MIXED path storage — pre-upgrade items store absolute
# /music/... paths, post-upgrade imports store library-relative paths. No
# single path:/path:: query form matches both populations (this is what made
# every dedup DELETE a silent no-op for two weeks). Item ids are storage-
# format-proof.
process_group() {
  local label="$1"; shift

  local ranked=""
  local hp id entry
  for entry in "$@"; do
    [[ -z "$entry" ]] && continue
    if [[ "$entry" == *"$SEP"* ]]; then
      id="${entry%%"$SEP"*}"
      hp=$(host_path "${entry#*"$SEP"}")
    else
      id=""
      hp="$entry"
    fi
    [[ -f "$hp" ]] || continue
    local score; score=$(rank_file "$hp")
    ranked+="${score}${SEP}${id}${SEP}${hp}"$'\n'
  done

  # No files on disk for this group — nothing to do. Return 0 (not 1): callers
  # invoke process_group as a standalone statement, so under `set -e` a non-zero
  # return would abort the whole run just because one group's files were gone.
  [[ -z "$ranked" ]] && return 0

  log ""
  log "--- $label ---"

  # Pass name for JSON output, matching dedup_candidates.pass_name's
  # convention (numbered_sibling|case_insensitive|normalized|cross_album_fuzzy).
  local pass_name="unknown"
  case "$label" in
    "P1 numbered-sibling"*) pass_name="numbered_sibling" ;;
    "P2 case-insensitive"*) pass_name="case_insensitive" ;;
    "P3 normalized"*)       pass_name="normalized" ;;
    "P4 cross-album"*)      pass_name="cross_album_fuzzy" ;;
  esac

  local first=1
  local keep_path="" keep_id="" any_auto=0
  while IFS="$SEP" read -r _sk id hp; do
    [[ -z "$hp" ]] && continue
    local size_bytes; size_bytes=$(stat -c '%s' "$hp" 2>/dev/null || echo 0)
    local size_h; size_h=$(numfmt --to=iec --suffix=B "$size_bytes")
    if [[ $first -eq 1 ]]; then
      log "  KEEP   ($size_h) $hp"
      keep_path="$hp"
      keep_id="$id"
      first=0
    else
      log "  DELETE ($size_h) $hp"

      # Auto-apply-numbered deletes numbered_sibling matches unconditionally,
      # bypassing --only-paths (that gate exists only for the manual confirm
      # flow, which never runs alongside this flag). All other passes stay
      # dry-run/log-only here regardless.
      local do_delete=0 auto_applied=0
      if [[ $APPLY -eq 1 ]]; then
        do_delete=1
      elif [[ $AUTO_APPLY_NUMBERED -eq 1 ]] \
        && { is_numbered_twin "$keep_path" "$hp" || is_format_upgrade "$pass_name" "$keep_path" "$hp"; }; then
        do_delete=1
        auto_applied=1
        any_auto=1
      fi
      local auto_json="false"; [[ $auto_applied -eq 1 ]] && auto_json="true"
      json_emit "{\"pass\":\"${pass_name}\",\"keep_path\":\"$(json_escape "$keep_path")\",\"delete_path\":\"$(json_escape "$hp")\",\"delete_size_bytes\":${size_bytes},\"auto_applied\":${auto_json}}"

      if [[ $do_delete -eq 1 ]]; then
        # With --only-paths given, only delete entries the caller explicitly
        # confirmed; without it, delete everything (original behavior).
        if [[ $auto_applied -eq 0 && -n "$ONLY_PATHS_FILE" && -z "${ONLY_PATHS[$hp]:-}" ]]; then
          log "    (skipped — not in --only-paths confirm list)"
        elif [[ -n "$id" ]]; then
          # In beets — beet remove -d removes from DB + disk. Best-effort: a
          # single failed removal is logged, not fatal, so the rest of the
          # confirmed deletions still proceed.
          beet remove -d -f "id:${id}" >> "$LOG" 2>&1 || log "    WARN: beet remove id:${id} failed"
        else
          # Ghost file — only on disk; just rm
          rm -f "$hp" 2>>"$LOG" || log "    WARN: rm failed for $hp"
        fi
      fi
    fi
  done < <(echo "$ranked" | grep -v '^$' | sort -n)

  # An auto-applied numbered-twin deletion can leave the numbered-named file
  # as the sole survivor (it won on size despite the dirty-filename penalty).
  # Rename it back to the canonical name so the library doesn't accumulate
  # ".1."/".2." filenames for tracks that no longer have a duplicate. Only
  # auto_applied triggers this (never a manual --only-paths confirm, which
  # may leave the sibling undeleted if the user didn't confirm it -- renaming
  # then would collide); only when it's actually in beets (a ghost survivor
  # has nothing to move); only when the name still has the dirty infix.
  if [[ $any_auto -eq 1 && -n "$keep_id" && "$keep_path" =~ \.[0-9]+\.[A-Za-z0-9]+$ ]]; then
    beet move "id:${keep_id}" >> "$LOG" 2>&1 || log "    WARN: beet move id:${keep_id} failed (rename survivor)"
  fi
  return 0  # explicit: the loop's last command status must not leak out under set -e
}

# ============================================================
# Pass 1: numbered siblings
# Find every *.N.ext in beets where *.ext also exists on disk.
# ============================================================
log ""
log "[$(date -Iseconds)] === Pass 1: numbered siblings ==="

# Dump id + display path from beets once. beets normalizes $path for display
# (real /data/music/Library/... paths since the 2026-07-08 rewrite; /music/...
# on any legacy entry) even though the DB stores a mix of absolute and
# relative — so the display paths are safe to compare/convert via host_path(),
# and the ids are what we hand to beet remove/move.
beet ls -f "\$id${SEP}\$path" 2>/dev/null > /tmp/beets-id-paths.txt
cut -d"$SEP" -f2- /tmp/beets-id-paths.txt > /tmp/beets-all-paths.txt

while IFS="$SEP" read -r numbered_id numbered_cp; do
  [[ -z "$numbered_cp" ]] && continue

  ext="${numbered_cp##*.}"
  canonical_cp=$(echo "$numbered_cp" | sed -E "s/\.[0-9]+\.${ext}$/.${ext}/")
  [[ "$canonical_cp" == "$numbered_cp" ]] && continue  # no .N. found

  host_numbered=$(host_path "$numbered_cp")
  host_canonical=$(host_path "$canonical_cp")

  [[ -f "$host_numbered" ]] || continue
  [[ -f "$host_canonical" ]] || continue

  # If the canonical is also in beets, Pass 2 handles this pair via beet remove -d.
  # Skip here to avoid a partial rm that leaves a stale beets DB entry.
  grep -qxF "$canonical_cp" /tmp/beets-all-paths.txt && continue

  score_numbered=$(rank_file "$host_numbered")
  score_canonical=$(rank_file "$host_canonical")

  if [[ $score_canonical -le $score_numbered ]]; then
    # Canonical wins: delete numbered (in beets), keep canonical (ghost)
    process_group "P1 numbered-sibling: ${numbered_cp##*/Library/}" \
      "$host_canonical" "${numbered_id}${SEP}${numbered_cp}"
    if [[ $APPLY -eq 1 || $AUTO_APPLY_NUMBERED -eq 1 ]]; then
      # canonical is now a ghost file; import it into beets
      beet import -q -s "${canonical_cp}" >> "$LOG" 2>&1 || log "  WARN: beet import failed for ${canonical_cp}"
    fi
  else
    # Numbered wins (canonical is ghost): rm the ghost, then beet move to rename
    # the numbered file to the canonical path (id query — see process_group).
    # Safe to reach with just AUTO_APPLY_NUMBERED: this loop only ever builds
    # genuine numbered-sibling pairs (see is_numbered_twin), and process_group
    # above already deleted the ghost canonical before this runs, so the move
    # target is guaranteed clear -- no rename collision.
    process_group "P1 numbered-sibling: ${numbered_cp##*/Library/}" \
      "${numbered_id}${SEP}${numbered_cp}" "$host_canonical"
    if [[ $APPLY -eq 1 || $AUTO_APPLY_NUMBERED -eq 1 ]]; then
      beet move "id:${numbered_id}" >> "$LOG" 2>&1 || log "  WARN: beet move id:${numbered_id} failed"
    fi
  fi
done < <(grep -E '\.[0-9]+\.(flac|mp3|m4a|ogg|opus|wav)$' /tmp/beets-id-paths.txt)

P1_GROUPS=$(grep -c '^--- P1 ' "$LOG" 2>/dev/null; true)
log ""
log "Pass 1 complete: $P1_GROUPS numbered-sibling groups"

# ============================================================
# Pass 2: case-insensitive beet-level dedup
# Re-fetch track list so Pass 1 deletions are reflected.
# ============================================================
log ""
log "[$(date -Iseconds)] === Pass 2: case-insensitive title/album/artist dedup ==="

beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
  2>/dev/null > /tmp/all-tracks.txt

gawk -F"${SEP}" -v sep="$SEP" '
# Sentinel titles carry zero identity — multiple distinct tracks can land on
# the same value when spotify-retag misses and the Soulseek source had a
# blank/placeholder title. Grouping by sentinel title = deleting different
# tracks that happened to share a placeholder. Common offenders:
#   ""                       fully empty title
#   "[unknown]" "unknown"    sldl/beets fallback
#   "track 1" .. "track NN"  stock CD-rip placeholder
function is_sentinel_title(t,    lt) {
  lt = tolower(t)
  gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
  if (lt == "") return 1
  if (lt == "[unknown]" || lt == "unknown") return 1
  if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
  return 0
}
{
  if (is_sentinel_title($4)) next
  key = tolower($2) sep tolower($3) sep tolower($4)
  paths[key] = paths[key] $1 sep $5 "\n"
  count[key]++
}
END {
  for (k in paths) {
    if (count[k] >= 2) print "===" k "\n" paths[k] "---"
  }
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt

GROUP_KEY=""
GROUP_PATHS=()

flush_p2_group() {
  [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
  process_group "P2 case-insensitive: $GROUP_KEY" "${GROUP_PATHS[@]}"
  GROUP_PATHS=()
}

while IFS= read -r line; do
  if [[ "$line" == ===* ]]; then
    flush_p2_group
    GROUP_KEY="${line#===}"
    GROUP_PATHS=()
  elif [[ "$line" == "---" ]]; then
    flush_p2_group
  elif [[ -n "$line" ]]; then
    GROUP_PATHS+=("$line")
  fi
done < /tmp/dups-grouped.txt
flush_p2_group  # flush last group

P2_GROUPS=$(grep -c '^--- P2 ' "$LOG" 2>/dev/null; true)
log ""
log "Pass 2 complete: $P2_GROUPS case-insensitive dup groups"

# ============================================================
# Pass 3: normalized-key dedup (strip non-alphanumerics)
# Re-fetch track list so Pass 2 deletions are reflected.
# ============================================================
log ""
log "[$(date -Iseconds)] === Pass 3: normalized title/album/artist dedup ==="

beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
  2>/dev/null > /tmp/all-tracks.txt

# Normalize: lowercase + strip everything except a-z 0-9. Group by the
# concatenation. Suppresses dupe pairs already caught by Pass 2 (same exact
# key) by also tracking the raw key per group and skipping single-key groups.
gawk -F"${SEP}" -v sep="$SEP" '
function norm(s,    t) {
  t = tolower(s)
  gsub(/[^a-z0-9]/, "", t)
  return t
}
function is_sentinel_title(t,    lt) {
  lt = tolower(t)
  gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
  if (lt == "") return 1
  if (lt == "[unknown]" || lt == "unknown") return 1
  if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
  return 0
}
{
  if (is_sentinel_title($4)) next
  raw = tolower($2) sep tolower($3) sep tolower($4)
  na = norm($2); nb = norm($3); nc = norm($4)
  # Title is the only field guaranteed unique per track. If it normalizes
  # to empty (all unicode/punctuation), refuse to group — would cause false
  # positives across unrelated unicode-titled tracks. A 1-char normalized
  # title is fine as long as artist+album also normalize non-empty
  # (covers "ID"/"OP"/single-letter tracks; rejects all-unicode titles
  # that collapse to "").
  if (length(nc) == 0) next
  if (length(nc) < 3 && (length(na) == 0 || length(nb) == 0)) next
  k = na sep nb sep nc
  paths[k] = paths[k] $1 sep $5 "\n"
  count[k]++
  raws[k][raw] = 1
}
END {
  for (k in paths) {
    if (count[k] < 2) continue
    # Skip groups that collapse to a single raw key — those were Pass 2 dups.
    raw_count = 0
    for (r in raws[k]) raw_count++
    if (raw_count < 2) continue
    print "===" k "\n" paths[k] "---"
  }
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt

GROUP_KEY=""
GROUP_PATHS=()

flush_p3_group() {
  [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
  process_group "P3 normalized: $GROUP_KEY" "${GROUP_PATHS[@]}"
  GROUP_PATHS=()
}

while IFS= read -r line; do
  if [[ "$line" == ===* ]]; then
    flush_p3_group
    GROUP_KEY="${line#===}"
    GROUP_PATHS=()
  elif [[ "$line" == "---" ]]; then
    flush_p3_group
  elif [[ -n "$line" ]]; then
    GROUP_PATHS+=("$line")
  fi
done < /tmp/dups-grouped.txt
flush_p3_group  # flush last group

P3_GROUPS=$(grep -c '^--- P3 ' "$LOG" 2>/dev/null; true)
log ""
log "Pass 3 complete: $P3_GROUPS normalized dup groups"

# ============================================================
# Pass 4: cross-album dedup (album-agnostic key)
# Catches cases where the same track exists once as a single and once on a
# compilation/Beatport-style album — the album field always disagrees, so
# Pass 2/3 miss them. Keys on (artist_norm, title_base_norm) where title_base
# strips trailing mix-version suffixes ("(Extended Mix)", " - Radio Edit",
# bare "  Extended Mix" with double-space, etc.).
#
# Allowlist: artists who legitimately have the same track on multiple albums
# (live + studio + remix etc.) — these are SKIPPED in Pass 4. Add to
# CROSS_ALBUM_KEEP_RE or the album-pattern skip below if needed.
# ============================================================
log ""
log "[$(date -Iseconds)] === Pass 4: cross-album dedup (artist+title-base) ==="

beet ls -f "\$id${SEP}\$albumartist${SEP}\$album${SEP}\$title${SEP}\$path" \
  2>/dev/null > /tmp/all-tracks.txt

# Pass artist + album allowlists into awk via -v.
# The artist keep-list (artists whose same-title tracks across albums are NOT
# treated as duplicates) is user-editable config, not baked in here. Build the
# anchored regex from cross-album-keep.list: normalize each name the same way
# the awk norm() does (lowercase, letters+digits only) and OR them together.
# An empty/absent list means "protect nobody" -> use a pattern that can never
# match a normalized name (which only ever contains [a-z0-9], never '_').
CROSS_ALBUM_KEEP_FILE="${ALEMBIC_CONFIG_DIR:-/config}/pipeline/cross-album-keep.list"
if [[ -f "$CROSS_ALBUM_KEEP_FILE" ]]; then
  # `|| true`: with an all-comments file (the default), the first grep matches
  # nothing and returns non-zero, which under set -o pipefail + set -e would
  # otherwise abort here. An empty result is the correct "protect nobody" case.
  _keep_names=$(grep -vE '^[[:space:]]*(#|$)' "$CROSS_ALBUM_KEEP_FILE" \
    | tr 'A-Z' 'a-z' | sed -E 's/[^a-z0-9]//g' | grep -v '^$' | paste -sd'|' - || true)
fi
if [[ -n "${_keep_names:-}" ]]; then
  CROSS_ALBUM_KEEP_RE="^(${_keep_names})\$"
else
  CROSS_ALBUM_KEEP_RE='_never_'
fi
# Album-name patterns that indicate "intentional alt version" — these rows are
# filtered out before grouping, so any group needing 2+ matches will only form
# from items whose albums DON'T look like remix/live/single-version releases.
# Tagging convention: when the user has multiple legitimate versions of a
# track, the alt-version copies live on albums with explicit version markers
# in the album title ("(Extended Mix) - Single", "(Remixed)", "Live at X").
# Without this filter, Pass 4 would try to dedupe Qrion's "Proud" off the
# "I Hope It Lasts Forever (Remixed)" album, Fred again._ studio tracks
# against "Apple Music Live" recordings, etc.
ALBUM_KEEP_RE='(soundtrack|originalmotionpicture|ost|vgm|gameost|liveat|livein|livefrom|livesession|applemusiclive|applelive|bbclive|bbcsession|mtvunplugged|unplugged|concert|inconcert|extended|remix|remixed|remixes|originalmix|radioedit|radiomix|clubmix|djedit|djmix|dubmix|vocalmix|instrumentalmix|longversion|shortversion|albumversion|originalversion)'

gawk -F"${SEP}" -v sep="$SEP" \
    -v artist_keep="$CROSS_ALBUM_KEEP_RE" \
    -v album_keep="$ALBUM_KEEP_RE" '
function norm(s,    t) {
  t = tolower(s)
  gsub(/[^a-z0-9]/, "", t)
  return t
}
# Strip trailing mix-version suffix from title so "Foo" and "Foo (Extended Mix)"
# share the same base. Only strips generic version markers — does NOT strip
# remixer-named suffixes like "(Joris Voorn Remix)", which are different songs.
function is_sentinel_title(t,    lt) {
  lt = tolower(t)
  gsub(/^[[:space:]]+|[[:space:]]+$/, "", lt)
  if (lt == "") return 1
  if (lt == "[unknown]" || lt == "unknown") return 1
  if (lt ~ /^track[[:space:]]*[0-9]+$/) return 1
  return 0
}
function strip_mix_suffix(s,    t, sfx) {
  t = tolower(s)
  sfx = "(extended[[:space:]]*mix|extended[[:space:]]*version|extended|" \
        "radio[[:space:]]*edit|radio[[:space:]]*mix|radio[[:space:]]*version|" \
        "original[[:space:]]*mix|original[[:space:]]*version|" \
        "club[[:space:]]*mix|dj[[:space:]]*edit|dj[[:space:]]*mix|" \
        "long[[:space:]]*version|short[[:space:]]*version|" \
        "single[[:space:]]*version|album[[:space:]]*version)"
  # parenthetical: " (Extended Mix)"
  t = gensub("[[:space:]]*\\(" sfx "\\)[[:space:]]*$", "", "g", t)
  # dash-separated: " - Extended Mix"
  t = gensub("[[:space:]]*-[[:space:]]+" sfx "[[:space:]]*$", "", "g", t)
  # bare trailing: "  Extended Mix"
  t = gensub("[[:space:]]+" sfx "[[:space:]]*$", "", "g", t)
  return t
}
{
  id = $1; artist = $2; album = $3; title = $4; path = $5
  if (is_sentinel_title(title)) next
  na = norm(artist)
  if (na ~ artist_keep) next
  if (norm(album) ~ album_keep) next
  tb = strip_mix_suffix(title)
  ntb = norm(tb)
  if (length(ntb) < 4) next
  k = na sep ntb
  paths[k] = paths[k] id sep path "\n"
  count[k]++
  raws3[k][na sep norm(album) sep norm(title)] = 1
  albums[k][album] = 1
}
END {
  for (k in paths) {
    if (count[k] < 2) continue
    # Skip if all items share the same Pass 3 normalized key (Pass 3 already handled).
    rcount = 0; for (r in raws3[k]) rcount++
    if (rcount < 2) continue
    # Require ≥2 distinct album names — otherwise Pass 3 would have grouped them.
    acount = 0; for (a in albums[k]) acount++
    if (acount < 2) continue
    print "===" k "\n" paths[k] "---"
  }
}' /tmp/all-tracks.txt > /tmp/dups-grouped.txt

GROUP_KEY=""
GROUP_PATHS=()

flush_p4_group() {
  [[ ${#GROUP_PATHS[@]} -lt 2 ]] && { GROUP_PATHS=(); return; }
  process_group "P4 cross-album: $GROUP_KEY" "${GROUP_PATHS[@]}"
  GROUP_PATHS=()
}

while IFS= read -r line; do
  if [[ "$line" == ===* ]]; then
    flush_p4_group
    GROUP_KEY="${line#===}"
    GROUP_PATHS=()
  elif [[ "$line" == "---" ]]; then
    flush_p4_group
  elif [[ -n "$line" ]]; then
    GROUP_PATHS+=("$line")
  fi
done < /tmp/dups-grouped.txt
flush_p4_group  # flush last group

P4_GROUPS=$(grep -c '^--- P4 ' "$LOG" 2>/dev/null; true)
log ""
log "Pass 4 complete: $P4_GROUPS cross-album dup groups"

rm -f /tmp/beets-all-paths.txt /tmp/beets-id-paths.txt /tmp/all-tracks.txt /tmp/dups-grouped.txt

TOTAL_GROUPS=$(( P1_GROUPS + P2_GROUPS + P3_GROUPS + P4_GROUPS ))
KEPT=$(grep -c '^  KEEP   ' "$LOG" 2>/dev/null; true)
DELETED=$(grep -c '^  DELETE ' "$LOG" 2>/dev/null; true)

log ""
log "=== Summary: $TOTAL_GROUPS dup groups ($P1_GROUPS numbered-sibling, $P2_GROUPS case-insensitive, $P3_GROUPS normalized, $P4_GROUPS cross-album), $KEPT kept, $DELETED to delete ==="
[[ $APPLY -eq 0 ]] && log "DRY RUN — re-run with --apply to actually delete"

# Explicit success. Without this the script's exit status is the previous
# line's, which is non-zero in --apply mode (the `[[ $APPLY -eq 0 ]]` test
# fails), making a successful apply look like a failed run to the caller.
# With set -e, any real failure above aborts before reaching here.
exit 0