Stop the daily status report (and its Telegram digest) being starved by the lock
Root cause of a missed 9am notification: the read-only pipeline_status_report runs at 09:30, but enrich_buy_url (09:10, --apply) held the single global pipeline lock long past 09:30, so the status report recorded skipped_lock and never sent its Telegram digest. build_fingerprint_index (09:25) was starved the same way. - run_job/run_lib_script gain use_lock; the status report runs with use_lock=False since it only reads logs, pings Navidrome, and sends Telegram. It can no longer be blocked by a long write job, and running concurrently is safe. - _lib was silently dropping its timeout argument, so maintenance jobs had no timeout and a hung one held the lock until the next container restart. Thread timeout through _lib -> run_lib_script -> run_job, cap enrich_buy_url at 30m and build_fingerprint_index at 60m, and give the status report a 5m cap. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -62,28 +62,37 @@ async def run_job(
|
||||
argv: list[str],
|
||||
triggered_by: str = "schedule",
|
||||
timeout: float | None = None,
|
||||
use_lock: bool = True,
|
||||
) -> JobRun:
|
||||
"""Run a pipeline command under the shared mutual-exclusion lock,
|
||||
recording one job_runs row start-to-finish. If the lock is already
|
||||
held, records status='skipped_lock' immediately and returns without
|
||||
running anything -- the old flock -n behavior, now visible in the UI
|
||||
instead of silently skipping."""
|
||||
instead of silently skipping.
|
||||
|
||||
use_lock=False runs the command WITHOUT taking the pipeline lock, for
|
||||
read-only jobs that never touch the library (e.g. the status report).
|
||||
Those must never be starved by a long-running write job, and running
|
||||
them concurrently is safe."""
|
||||
started_at = time.time()
|
||||
|
||||
if not await _try_acquire_nowait():
|
||||
db = SessionLocal()
|
||||
run = JobRun(
|
||||
job_key=job_key,
|
||||
started_at=started_at,
|
||||
finished_at=started_at,
|
||||
status="skipped_lock",
|
||||
triggered_by=triggered_by,
|
||||
)
|
||||
db.add(run)
|
||||
db.commit()
|
||||
db.refresh(run)
|
||||
db.close()
|
||||
return run
|
||||
acquired = False
|
||||
if use_lock:
|
||||
if not await _try_acquire_nowait():
|
||||
db = SessionLocal()
|
||||
run = JobRun(
|
||||
job_key=job_key,
|
||||
started_at=started_at,
|
||||
finished_at=started_at,
|
||||
status="skipped_lock",
|
||||
triggered_by=triggered_by,
|
||||
)
|
||||
db.add(run)
|
||||
db.commit()
|
||||
db.refresh(run)
|
||||
db.close()
|
||||
return run
|
||||
acquired = True
|
||||
|
||||
try:
|
||||
log_dir = settings.logs_dir / job_key.replace(":", "_")
|
||||
@@ -140,7 +149,8 @@ async def run_job(
|
||||
db.close()
|
||||
return run
|
||||
finally:
|
||||
_lock.release()
|
||||
if acquired:
|
||||
_lock.release()
|
||||
|
||||
|
||||
async def run_job_capture(
|
||||
@@ -248,9 +258,22 @@ async def run_import_track(args: list[str], triggered_by: str = "manual") -> Job
|
||||
return await run_job("manual:import", [str(script)] + args, triggered_by=triggered_by)
|
||||
|
||||
|
||||
async def run_lib_script(job_key: str, script_name: str, args: list[str] | None = None, triggered_by: str = "schedule") -> JobRun:
|
||||
async def run_lib_script(
|
||||
job_key: str,
|
||||
script_name: str,
|
||||
args: list[str] | None = None,
|
||||
triggered_by: str = "schedule",
|
||||
timeout: float | None = None,
|
||||
use_lock: bool = True,
|
||||
) -> JobRun:
|
||||
script = settings.pipeline_dir / "lib" / script_name
|
||||
return await run_job(job_key, [str(script)] + (args or []), triggered_by=triggered_by)
|
||||
return await run_job(
|
||||
job_key,
|
||||
[str(script)] + (args or []),
|
||||
triggered_by=triggered_by,
|
||||
timeout=timeout,
|
||||
use_lock=use_lock,
|
||||
)
|
||||
|
||||
|
||||
async def run_beet(job_key: str, args: list[str], triggered_by: str = "schedule") -> JobRun:
|
||||
|
||||
Reference in New Issue
Block a user