fix: P0-4 schedule node janitor + runtime reaper/lock invariants
S2: schedule worker add janitor task that force-terminals node_runs whose deadline (timeout_seconds + retry_count*retry_interval + 120s slack from started_at) has passed. Closes the gap where outbox retry exhaustion (5 tries, capped 30s backoff) marked the *event* failed but left the *node_run* stuck in queued/running forever. Re-reads the row under FOR UPDATE before writing so a worker that races us to a real terminal state is not overwritten; idempotency key uses :timed_out variant so the :finished path cannot collide. R1: extract _reap_once() from _reap_loop for testability; in the dead-process branch, re-verify (process.pid, started_at) against the live JUPYTER_PROCESSES entry before del. A start_workspace that replaced the dead record mid-cycle used to have its new entry silently erased by the reaper's stale snapshot — leaked the port. R2: delete _drop_workspace_lock and its two call sites (stop_workspace tail, get_workspace 404 path). Popping the lock object after release breaks mutual exclusion for any coroutine still holding the old reference while a fresh caller gets a new lock object — same ws_id can race two starts. The dict is bounded by the number of workspaces so the leak is negligible; invariant lives on WORKSPACE_LOCKS in a comment. Tests: - schedule/tests/test_janitor.py — 8 tests covering normal kill / healthy-skip / worker-race / never-started / multi-row batch / cancellation propagation / per-iteration self-heal - runtime/tests/test_process.py — 7 tests covering reaper identity match / replacement-skip / alive-preserved + lock same-object / concurrent-serialize / survives-stop / helper-removed guard uv run --package schedule pytest schedule/tests → 14 passed uv run --package runtime pytest runtime/tests → 7 passed Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
tao.chen
co-authored by
Claude Fable 5
Claude Fable 5
parent
e5633cc95a
commit
4acfbb162f
@@ -21,9 +21,10 @@ from common.db.models import (
|
||||
ConsumerInbox,
|
||||
OutboxEvents,
|
||||
ScheduleNodeRuns,
|
||||
ScheduleNodes,
|
||||
ScheduleRuns,
|
||||
)
|
||||
from common.eventing import add_outbox_event, schedule_event_type, utcnow
|
||||
from common.eventing import add_outbox_event, event_time, schedule_event_type, utcnow
|
||||
from common.ids import new_ulid
|
||||
from loguru import logger
|
||||
from sqlalchemy import select
|
||||
@@ -73,6 +74,27 @@ class DispatchOrchestrator:
|
||||
# terminal state before the poll re-picks it.
|
||||
LEASE_SLACK = timedelta(seconds=30)
|
||||
|
||||
# Margins used by the node-run janitor (see ``_janitor_loop``).
|
||||
#
|
||||
# ``NODE_JANITOR_GRACE_SECONDS`` is added on top of each node's
|
||||
# ``timeout_seconds + retry_count * retry_interval_sec`` when judging
|
||||
# whether a started-but-not-finished row is stuck. Absorbs the outbox
|
||||
# ``min(30, 2**retry_count)`` backoff + the lease slack above + a few
|
||||
# seconds of scheduling jitter. Tuned for the worst case the
|
||||
# orchestrator itself produces, so the janitor cannot race a legitimate
|
||||
# retry path to terminal state.
|
||||
NODE_JANITOR_GRACE_SECONDS = 120
|
||||
# Floor for ``started_at IS NULL`` rows (never dispatched). 1h rides
|
||||
# out an orchestrator restart that drops the lease mid-flight; past
|
||||
# that the row is dead and forcing a terminal state lets the DAG
|
||||
# advance.
|
||||
NODE_JANITOR_QUEUED_GRACE_SECONDS = 3600
|
||||
# Loop period + batch size. Detection lag = interval + the time it
|
||||
# takes to scan, currently well under a minute. 50 rows per cycle
|
||||
# keeps worst-case fan-out bounded.
|
||||
NODE_JANITOR_INTERVAL_SECONDS = 30
|
||||
NODE_JANITOR_BATCH = 50
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -88,6 +110,7 @@ class DispatchOrchestrator:
|
||||
self.dispatch_lock = asyncio.Lock()
|
||||
self._loop_task: asyncio.Task[None] | None = None
|
||||
self._exec_loop_task: asyncio.Task[None] | None = None
|
||||
self._janitor_task: asyncio.Task[None] | None = None
|
||||
self._exec_tasks: set[asyncio.Task[None]] = set()
|
||||
self._exec_semaphore = asyncio.Semaphore(execution_concurrency)
|
||||
|
||||
@@ -100,6 +123,10 @@ class DispatchOrchestrator:
|
||||
self._execution_loop(),
|
||||
name="scheduler-node-execute",
|
||||
)
|
||||
self._janitor_task = asyncio.create_task(
|
||||
self._janitor_loop(),
|
||||
name="scheduler-node-janitor",
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._loop_task is not None:
|
||||
@@ -116,6 +143,13 @@ class DispatchOrchestrator:
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._exec_loop_task = None
|
||||
if self._janitor_task is not None:
|
||||
self._janitor_task.cancel()
|
||||
try:
|
||||
await self._janitor_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._janitor_task = None
|
||||
if self._exec_tasks:
|
||||
await asyncio.gather(*self._exec_tasks, return_exceptions=True)
|
||||
self._exec_tasks.clear()
|
||||
@@ -148,6 +182,181 @@ class DispatchOrchestrator:
|
||||
logger.exception("node execute loop failed")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def _janitor_loop(self) -> None:
|
||||
"""Periodically force terminal state on stuck ``ScheduleNodeRuns``.
|
||||
|
||||
Original P0-4 S2 bug: when ``job.node.execute`` outbox retries are
|
||||
exhausted (5 tries, capped 30s backoff) the *event* is marked
|
||||
``failed`` but no one reconciles the *node_run* — the row stays
|
||||
in ``queued`` or ``running`` forever, DAG children are never
|
||||
dispatched, and the whole run sits at "running" with no further
|
||||
progress.
|
||||
|
||||
A second, harder failure mode: the worker crashes (OOM /
|
||||
``kill -9`` / forgotten upload) mid-execution. The lease-based
|
||||
re-eligibility keeps re-dispatching the event, but a worker
|
||||
that's stuck without writing a terminal state is invisible to
|
||||
the retry counter — it just keeps timing out.
|
||||
|
||||
Both collapse into one wall-clock condition: ``started_at +
|
||||
`` budget < now()`` (or, more rarely, ``started_at IS NULL`` for
|
||||
long enough). The janitor enforces that condition so no row
|
||||
can outlive its deadline regardless of which subsystem let go.
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
killed = await self._reap_stuck_node_runs()
|
||||
if killed:
|
||||
logger.warning(
|
||||
"node janitor reaped {} stuck node run(s)", killed,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("node janitor loop failed")
|
||||
await asyncio.sleep(self.NODE_JANITOR_INTERVAL_SECONDS)
|
||||
|
||||
async def _reap_stuck_node_runs(self) -> int:
|
||||
"""Find stuck ``ScheduleNodeRuns`` rows and force-terminal them.
|
||||
|
||||
SQL does a coarse pre-filter (``created_at`` at least one hour
|
||||
old so we don't even look at fresh rows). Per-row budget is
|
||||
computed in Python because it depends on each row's
|
||||
``ScheduleNodes.timeout_seconds`` + ``retry_count`` +
|
||||
``retry_interval_sec``, which can vary row to row.
|
||||
|
||||
Returns the count of rows actually flipped to ``timed_out``;
|
||||
rows that were already terminal (worker got there first) are
|
||||
silently skipped, so two janitors running side by side stay
|
||||
idempotent.
|
||||
"""
|
||||
queued_cutoff = utcnow() - timedelta(
|
||||
seconds=self.NODE_JANITOR_QUEUED_GRACE_SECONDS,
|
||||
)
|
||||
async with session_scope(self.session_factory) as session:
|
||||
statement = (
|
||||
select(ScheduleNodeRuns, ScheduleNodes, ScheduleRuns)
|
||||
.join(
|
||||
ScheduleNodes,
|
||||
ScheduleNodes.node_id == ScheduleNodeRuns.node_id,
|
||||
)
|
||||
.join(
|
||||
ScheduleRuns,
|
||||
ScheduleRuns.run_id == ScheduleNodeRuns.run_id,
|
||||
)
|
||||
.where(
|
||||
ScheduleNodeRuns.node_status.in_(("queued", "running")),
|
||||
ScheduleNodeRuns.is_deleted == 0,
|
||||
ScheduleNodes.is_deleted == 0,
|
||||
ScheduleRuns.is_deleted == 0,
|
||||
ScheduleNodeRuns.created_at <= queued_cutoff,
|
||||
)
|
||||
.order_by(ScheduleNodeRuns.created_at)
|
||||
.limit(self.NODE_JANITOR_BATCH)
|
||||
)
|
||||
candidates = list((await session.execute(statement)).all())
|
||||
now = utcnow()
|
||||
killed = 0
|
||||
for node_run, node, run in candidates:
|
||||
if node_run.started_at is not None:
|
||||
budget = (
|
||||
node.timeout_seconds
|
||||
+ node.retry_count * node.retry_interval_sec
|
||||
+ self.NODE_JANITOR_GRACE_SECONDS
|
||||
)
|
||||
deadline = node_run.started_at + timedelta(
|
||||
seconds=budget,
|
||||
)
|
||||
if now <= deadline:
|
||||
# Healthy row that just happened to be in the
|
||||
# SQL pre-filter window; skip without writing.
|
||||
continue
|
||||
# ``started_at IS NULL`` rows fell through the coarse
|
||||
# ``created_at <= queued_cutoff`` filter, so we know
|
||||
# they are at least an hour old and never dispatched.
|
||||
if await self._force_terminal_node_run(
|
||||
session,
|
||||
node_run=node_run,
|
||||
run=run,
|
||||
reason="NODE_RUN_TIMEOUT",
|
||||
):
|
||||
killed += 1
|
||||
return killed
|
||||
|
||||
async def _force_terminal_node_run(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
node_run: ScheduleNodeRuns,
|
||||
run: ScheduleRuns,
|
||||
reason: str,
|
||||
) -> bool:
|
||||
"""Mark ``node_run`` as ``timed_out`` and enqueue a ``NODE_FINISHED_EVENT``.
|
||||
|
||||
Re-reads the row under ``with_for_update`` so a worker that
|
||||
reports the result concurrently can't be undone by a second
|
||||
``timed_out`` write (and vice versa). Returns ``True`` iff this
|
||||
call performed the terminal write; ``False`` if the row had
|
||||
already moved on (worker raced us, another janitor raced us).
|
||||
|
||||
The idempotency key is distinct from the worker-reported
|
||||
``:finished`` key so the same ``node_run`` won't produce two
|
||||
``NODE_FINISHED_EVENT`` rows if both paths fire. The DAG
|
||||
consumer is idempotent on its end (``_advance_run`` is a no-op
|
||||
when ``node_status`` is already terminal), so even if a stray
|
||||
duplicate slipped through it would self-heal.
|
||||
"""
|
||||
locked = await session.scalar(
|
||||
select(ScheduleNodeRuns)
|
||||
.where(ScheduleNodeRuns.node_run_id == node_run.node_run_id)
|
||||
.with_for_update()
|
||||
)
|
||||
if locked is None or locked.node_status in TERMINAL_NODE_STATES:
|
||||
return False
|
||||
finished_at = utcnow()
|
||||
started_or_created = locked.started_at or locked.created_at
|
||||
locked.node_status = "timed_out"
|
||||
locked.finished_at = finished_at
|
||||
locked.duration_ms = int(
|
||||
(finished_at - started_or_created).total_seconds() * 1000,
|
||||
)
|
||||
locked.error_code = reason
|
||||
locked.message = (
|
||||
"节点执行超时已自动终止"
|
||||
if locked.started_at is not None
|
||||
else "节点从未派发，调度超时已自动终止"
|
||||
)[:2000]
|
||||
locked.state_version += 1
|
||||
await add_outbox_event(
|
||||
session,
|
||||
event_type=NODE_FINISHED_EVENT,
|
||||
producer="schedule-janitor",
|
||||
trace_id=new_ulid(),
|
||||
aggregate_type="schedule_node_run",
|
||||
aggregate_id=locked.node_run_id,
|
||||
idempotency_key=(
|
||||
f"{locked.node_run_id}:{locked.attempt_no}:timed_out"
|
||||
),
|
||||
payload={
|
||||
"workspace_id": run.workspace_id,
|
||||
"run_id": locked.run_id,
|
||||
"node_run_id": locked.node_run_id,
|
||||
"node_id": locked.node_id,
|
||||
"versions_id": locked.versions_id,
|
||||
"attempt_no": locked.attempt_no,
|
||||
"node_status": locked.node_status,
|
||||
"exit_code": None,
|
||||
"started_at": event_time(locked.started_at),
|
||||
"finished_at": event_time(finished_at),
|
||||
"duration_ms": locked.duration_ms,
|
||||
"logs_object_id": None,
|
||||
"result_object_id": None,
|
||||
"error_code": reason,
|
||||
"error_message": locked.message,
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
async def _claim_execution_events(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user