Stage 0: migrate pipeline scripts from /opt/sldl, scaffold repo
Moves all ~30 pipeline scripts, configs, and the vendored sldl binary into this repo (source /opt/sldl left untouched). Removes all docker exec/docker compose dependencies now that beets and sldl run in-process/as a subprocess of this container instead of via soulbeet/on-demand sldl containers. Replaces hardcoded host paths, Navidrome credentials, and Spotify credential sourcing with env-var-driven paths and shared credential loaders. Adds Dockerfile, entrypoint.sh, requirements.txt, docker-compose.snippet.yml, and the initial app DB schema.
This commit is contained in:
andrew
Executable
+208
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
scrub-watermark-text.py — strip Soulseek-uploader watermark URLs from
|
||||
metadata text frames across the library. Some uploaders pollute every
|
||||
available field (COMM, USLT, TXXX, WXXX, even TCON/genre) with their
|
||||
domain, so a track might have 8+ watermark frames each pointing at
|
||||
electronicfresh.com / djsoundtop.com / etc.
|
||||
|
||||
Targets — for any frame that contains a URL or known watermark domain:
|
||||
MP3 (ID3v2):
|
||||
COMM: comments
|
||||
USLT: unsynchronized lyrics
|
||||
TXXX: user-defined text (kept if value is non-URL)
|
||||
WXXX, WCOM, WPAY, WPUB, WORS, WCOP: URL link frames (always stripped)
|
||||
TCOP: copyright
|
||||
TPUB: publisher
|
||||
TENC: encoded by
|
||||
TCON: genre (only stripped if it's a URL — real genres are short text)
|
||||
FLAC (Vorbis):
|
||||
COMMENT, DESCRIPTION, COPYRIGHT, ENCODED-BY, CONTACT, WEBPAGE,
|
||||
LICENSE — wiped if value contains URL pattern
|
||||
GENRE — wiped only if it looks like a URL
|
||||
|
||||
Frames that hold real metadata we always keep:
|
||||
TIT2/TIT1/TPE1/TPE2/TALB/TPOS/TRCK/TDRC/TYER/TXXX:GROUPING (ours)
|
||||
ARTIST/ALBUMARTIST/ALBUM/TITLE/TRACKNUMBER/DISCNUMBER/DATE/GROUPING
|
||||
|
||||
Usage:
|
||||
scrub-watermark-text.py # dry run
|
||||
scrub-watermark-text.py --apply # actually strip
|
||||
"""
|
||||
import sys, re, argparse
|
||||
from pathlib import Path
|
||||
from mutagen import File as MFile
|
||||
from mutagen.id3 import ID3, ID3NoHeaderError
|
||||
from mutagen.flac import FLAC
|
||||
|
||||
LIBRARY = f"{os.environ.get('MUSIC_DATA_DIR', '/data/music')}/Library"
|
||||
|
||||
# Anything containing one of these is suspect. Add to taste.
|
||||
WATERMARK_DOMAINS = [
|
||||
"electronicfresh", "djsoundtop", "iptorrents", "torrentday",
|
||||
"ftpdjemilio", "hypeddit", "hypeddit.com", "djseamusic", "djpool",
|
||||
"myzuka", "promodj",
|
||||
]
|
||||
|
||||
# Generic URL pattern — http(s)://, www., or a bare domain.tld
|
||||
URL_RE = re.compile(
|
||||
r"(?ix)"
|
||||
r"(?:https?://|www\.|[a-z0-9-]+\.(?:com|net|org|io|to|cc|me|tv|fm|gg|co|sh|ru))"
|
||||
r"[a-z0-9./?#=&_+:%-]*"
|
||||
)
|
||||
|
||||
# Frames that should be FULLY removed if they contain ANY URL.
|
||||
# NOTE: WCOM ("commercial information") and WPAY ("payment") are deliberately
|
||||
# NOT stripped — that's where the buy-link pipeline stores the legit Bandcamp
|
||||
# purchase URL (see sync-bandcamp.py BUY_URL_TAG / README "Buy links"). On
|
||||
# FLAC the equivalent COMMERCIAL_INFORMATION Vorbis comment is already safe
|
||||
# (it isn't in FLAC_TEXT_FIELDS_TO_CHECK). Watermarkers overwhelmingly abuse
|
||||
# COMM/TXXX/WXXX/USLT, not WCOM/WPAY, so excluding these costs us nothing.
|
||||
MP3_URL_LINK_FRAMES = ("WXXX", "WPUB", "WORS", "WCOP", "WOAR", "WOAS", "WOAF")
|
||||
# Frames where we strip the WHOLE frame if its text content contains a URL.
|
||||
MP3_TEXT_FRAMES_TO_CHECK = ("COMM", "USLT", "TCOP", "TPUB", "TENC", "TXXX",
|
||||
"WXXX", "TOPE", "TOAL")
|
||||
# TCON is special — only strip if it's a URL (real genres are normal text)
|
||||
|
||||
|
||||
def text_has_url(s):
|
||||
if not s: return False
|
||||
s = str(s)
|
||||
if URL_RE.search(s):
|
||||
return True
|
||||
sl = s.lower()
|
||||
return any(d in sl for d in WATERMARK_DOMAINS)
|
||||
|
||||
|
||||
def scrub_mp3(path, apply):
|
||||
try:
|
||||
tags = ID3(path)
|
||||
except (ID3NoHeaderError, Exception):
|
||||
return 0, []
|
||||
|
||||
to_delete = []
|
||||
actions = []
|
||||
|
||||
for k in list(tags.keys()):
|
||||
prefix = k[:4]
|
||||
# URL link frames — strip if they have anything URL-y
|
||||
if prefix in MP3_URL_LINK_FRAMES:
|
||||
v = getattr(tags[k], "url", None) or str(tags[k])
|
||||
if text_has_url(v):
|
||||
to_delete.append(k)
|
||||
actions.append(f"remove {k}: {str(v)[:80]}")
|
||||
continue
|
||||
# Text frames — check the rendered text
|
||||
if prefix in MP3_TEXT_FRAMES_TO_CHECK:
|
||||
v = str(tags[k])
|
||||
if text_has_url(v):
|
||||
to_delete.append(k)
|
||||
actions.append(f"remove {k}: {v[:80]}")
|
||||
continue
|
||||
# TCON: genre. Strip if it's clearly a URL.
|
||||
if prefix == "TCON":
|
||||
v = str(tags[k])
|
||||
if text_has_url(v):
|
||||
to_delete.append(k)
|
||||
actions.append(f"remove TCON (was URL'd genre): {v[:80]}")
|
||||
continue
|
||||
|
||||
if apply and to_delete:
|
||||
for k in to_delete:
|
||||
del tags[k]
|
||||
try:
|
||||
tags.save(path)
|
||||
except Exception as e:
|
||||
actions.append(f" SAVE FAILED: {e}")
|
||||
return len(to_delete), actions
|
||||
|
||||
|
||||
# Vorbis comment fields (case-insensitive in spec, we handle that)
|
||||
FLAC_TEXT_FIELDS_TO_CHECK = (
|
||||
"comment", "description", "copyright", "encoded-by",
|
||||
"contact", "webpage", "website", "license", "url",
|
||||
)
|
||||
|
||||
|
||||
def scrub_flac(path, apply):
|
||||
try:
|
||||
f = FLAC(path)
|
||||
except Exception:
|
||||
return 0, []
|
||||
|
||||
to_delete = [] # list of (field_name, value)
|
||||
actions = []
|
||||
|
||||
for key in list(f.tags.keys() if f.tags else []):
|
||||
kl = key.lower()
|
||||
vals = f.tags[key]
|
||||
if kl in FLAC_TEXT_FIELDS_TO_CHECK:
|
||||
for v in vals:
|
||||
if text_has_url(v):
|
||||
to_delete.append((key, v))
|
||||
actions.append(f"remove {key}: {v[:80]}")
|
||||
elif kl == "genre":
|
||||
for v in vals:
|
||||
if text_has_url(v):
|
||||
to_delete.append((key, v))
|
||||
actions.append(f"remove genre (was URL'd): {v[:80]}")
|
||||
|
||||
if apply and to_delete:
|
||||
# Group by field, rebuild the field's value list without URL-y entries.
|
||||
fields_to_rewrite = {}
|
||||
for key, val in to_delete:
|
||||
fields_to_rewrite.setdefault(key, []).append(val)
|
||||
for key, bad_vals in fields_to_rewrite.items():
|
||||
current = list(f.tags[key])
|
||||
keep = [v for v in current if v not in bad_vals]
|
||||
if keep:
|
||||
f.tags[key] = keep
|
||||
else:
|
||||
del f.tags[key]
|
||||
try:
|
||||
f.save()
|
||||
except Exception as e:
|
||||
actions.append(f" SAVE FAILED: {e}")
|
||||
return len(to_delete), actions
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
print(f"[scrub-text] mode={'APPLY' if args.apply else 'DRY RUN'}")
|
||||
print(f"[scrub-text] watermark domains: {WATERMARK_DOMAINS}\n")
|
||||
|
||||
files = list(Path(LIBRARY).rglob("*.mp3")) + list(Path(LIBRARY).rglob("*.flac"))
|
||||
print(f"[scrub-text] scanning {len(files)} files...\n")
|
||||
|
||||
total_frames = 0
|
||||
files_touched = 0
|
||||
for i, p in enumerate(files, 1):
|
||||
if i % 200 == 0:
|
||||
print(f" ...{i}/{len(files)}", flush=True)
|
||||
sfx = p.suffix.lower()
|
||||
if sfx == ".mp3":
|
||||
n, actions = scrub_mp3(str(p), args.apply)
|
||||
elif sfx == ".flac":
|
||||
n, actions = scrub_flac(str(p), args.apply)
|
||||
else:
|
||||
continue
|
||||
if n > 0:
|
||||
files_touched += 1
|
||||
total_frames += n
|
||||
print(f"\n {p.name} ({n} frame(s))")
|
||||
for a in actions[:5]:
|
||||
print(f" {a}")
|
||||
if len(actions) > 5:
|
||||
print(f" ...and {len(actions) - 5} more")
|
||||
|
||||
verb = "stripped" if args.apply else "would-strip"
|
||||
print(f"\n[scrub-text] {verb} {total_frames} frame(s) across {files_touched} file(s)")
|
||||
if not args.apply:
|
||||
print("[scrub-text] re-run with --apply to write changes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user