andrew
7a49e2d507
The Telegram digest reported healthy weekly jobs as "never run yet": its maintenance section still globbed for cron-era per-script log filenames (clean-index-*.log etc.) that jobs run through pipeline_runner no longer write. It now reads job_runs, the scheduler's own record, so it reports the last success (age + summary snippet from that run's log), flags a newer failed or lock-skipped attempt on the same line, and distinguishes "never succeeded (last attempt: skipped, lock busy)" from genuinely never scheduled. Also matched the clear-bad-genres snippet grep to the script's current summary wording, and dropped the now-unused newest_log helper. The DB also showed WHY two jobs had never run: on Sunday the 08:30 strip-mb-tags run took 10m19s while holding the shared pipeline lock, so strip-watermark-art (08:35) and scrub-watermark-text (08:40) hit flock-style instant skip and lost their only slot of the week -- the 08:40 job missed by 19 seconds. Scheduled runs now wait up to 30 minutes for the lock (SCHEDULED_LOCK_WAIT_SECONDS) and only then record skipped_lock, so fixed-time blocks queue instead of starving; manual "Run now" keeps the instant skip since a person expects an immediate answer. Tests: scheduled-run queueing, lock-wait timeout, and manual instant-skip. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
113 lines
3.3 KiB
Python
113 lines
3.3 KiB
Python
import asyncio
|
|
|
|
from app.services import pipeline_runner as pr
|
|
|
|
# One loop for the whole module: pipeline_runner._lock is a module-level
|
|
# asyncio.Lock, and reusing it across different loops (as asyncio.run would
|
|
# create) raises "bound to a different event loop".
|
|
_loop = asyncio.new_event_loop()
|
|
|
|
|
|
def _run(coro):
|
|
return _loop.run_until_complete(coro)
|
|
|
|
|
|
def test_run_job_success_records_row_and_log():
|
|
run = _run(pr.run_job("test:ok", ["true"]))
|
|
assert run.status == "success"
|
|
assert run.exit_code == 0
|
|
assert run.log_path # a log file path was recorded
|
|
|
|
|
|
def test_run_job_failure():
|
|
run = _run(pr.run_job("test:fail", ["false"]))
|
|
assert run.status == "failed"
|
|
assert run.exit_code == 1
|
|
|
|
|
|
def test_manual_run_skips_immediately_when_lock_held():
|
|
async def scenario():
|
|
await pr._lock.acquire()
|
|
try:
|
|
return await pr.run_job("test:locked", ["true"], use_lock=True, triggered_by="manual")
|
|
finally:
|
|
pr._lock.release()
|
|
|
|
run = _run(scenario())
|
|
assert run.status == "skipped_lock"
|
|
|
|
|
|
def test_scheduled_run_waits_for_lock_by_default(monkeypatch):
|
|
# scheduled runs must QUEUE behind a held lock (bounded), not skip --
|
|
# instant-skip starved the Sunday maintenance block (see
|
|
# SCHEDULED_LOCK_WAIT_SECONDS)
|
|
monkeypatch.setattr(pr, "SCHEDULED_LOCK_WAIT_SECONDS", 5.0)
|
|
|
|
async def scenario():
|
|
await pr._lock.acquire()
|
|
task = asyncio.ensure_future(pr.run_job("test:queued", ["true"], triggered_by="schedule"))
|
|
await asyncio.sleep(0.2) # let the job start waiting on the lock
|
|
pr._lock.release()
|
|
return await task
|
|
|
|
run = _run(scenario())
|
|
assert run.status == "success"
|
|
assert not pr._lock.locked()
|
|
|
|
|
|
def test_scheduled_run_skips_after_lock_wait_timeout():
|
|
async def scenario():
|
|
await pr._lock.acquire()
|
|
try:
|
|
return await pr.run_job("test:waited_out", ["true"], triggered_by="schedule", lock_wait=0.2)
|
|
finally:
|
|
pr._lock.release()
|
|
|
|
run = _run(scenario())
|
|
assert run.status == "skipped_lock"
|
|
|
|
|
|
def test_use_lock_false_bypasses_held_lock():
|
|
async def scenario():
|
|
await pr._lock.acquire()
|
|
try:
|
|
return await pr.run_job("test:bypass", ["true"], use_lock=False)
|
|
finally:
|
|
pr._lock.release()
|
|
|
|
run = _run(scenario())
|
|
assert run.status == "success"
|
|
|
|
|
|
def test_lock_released_after_run():
|
|
_run(pr.run_job("test:release", ["true"]))
|
|
assert not pr._lock.locked()
|
|
|
|
|
|
def test_timeout_kills_and_marks_failed():
|
|
run = _run(pr.run_job("test:timeout", ["sleep", "5"], timeout=0.3))
|
|
assert run.status == "failed"
|
|
assert run.exit_code == -1
|
|
|
|
|
|
def test_run_job_capture_returns_and_persists_output():
|
|
from pathlib import Path
|
|
|
|
run, out = _run(pr.run_job_capture("test:cap", ["sh", "-c", "echo hello; echo world"]))
|
|
assert run.status == "success"
|
|
assert "hello" in out and "world" in out
|
|
assert "hello" in Path(run.log_path).read_text() # also written to the log file
|
|
|
|
|
|
def test_run_job_capture_skips_when_locked():
|
|
async def scenario():
|
|
await pr._lock.acquire()
|
|
try:
|
|
return await pr.run_job_capture("test:caplock", ["true"])
|
|
finally:
|
|
pr._lock.release()
|
|
|
|
run, out = _run(scenario())
|
|
assert run.status == "skipped_lock"
|
|
assert out == ""
|