修复失败策略运行逻辑

This commit is contained in:
Winnie
2026-08-18 11:10:10 +08:00
parent ff700d8db4
commit 3b0229f018
+9 -43
View File
@@ -553,6 +553,7 @@ class DispatchOrchestrator:
latest[row.node_id] = row latest[row.node_id] = row
max_concurrency = max(1, int(snapshot.get("max_concurrency", 1))) max_concurrency = max(1, int(snapshot.get("max_concurrency", 1)))
failure_policy = snapshot.get("failure_policy", "stop")
while True: while True:
changed = False changed = False
active_count = sum( active_count = sum(
@@ -586,7 +587,7 @@ class DispatchOrchestrator:
for item in latest.values() for item in latest.values()
) )
stop_all = ( stop_all = (
snapshot.get("failure_policy", "stop") == "stop" failure_policy == "stop"
and exhausted_failure and exhausted_failure
) )
for node in nodes: for node in nodes:
@@ -594,29 +595,15 @@ class DispatchOrchestrator:
if node_id in latest: if node_id in latest:
continue continue
parent_runs = [latest.get(parent) for parent in parents[node_id]] parent_runs = [latest.get(parent) for parent in parents[node_id]]
# Decide whether this node should be skipped. A node # 只有 ``stop`` 策略才会在失败后跳过尚未启动的节点。
# is only skipped when we know it can never run: # ``continue`` 表示“前一个节点失败也继续往后执行”,因此
# * ``stop_all`` — the whole run was aborted on the # 下游只需等待所有上游结束,不要求它们全部成功。
# first failure, so any not-yet-dispatched node is
# dropped;
# * all parents are terminal AND at least one
# failed — there is no remaining success path.
# If even one parent is still ``queued`` or
# ``running`` we keep waiting: under ``failure_policy
# == 'continue'`` a sibling might still succeed and
# the failed parent does not block that.
parents_terminal = all( parents_terminal = all(
item is not None item is not None
and item.node_status in TERMINAL_NODE_STATES and item.node_status in TERMINAL_NODE_STATES
for item in parent_runs for item in parent_runs
) )
any_parent_failed = any( if stop_all:
item is not None
and item.node_status in FAILED_NODE_STATES
for item in parent_runs
)
parents_blocked = parents_terminal and any_parent_failed
if stop_all or parents_blocked:
skipped = ScheduleNodeRuns( skipped = ScheduleNodeRuns(
node_run_id=new_ulid(), node_run_id=new_ulid(),
run_id=run.run_id, run_id=run.run_id,
@@ -627,11 +614,7 @@ class DispatchOrchestrator:
state_version=1, state_version=1,
finished_at=utcnow(), finished_at=utcnow(),
duration_ms=0, duration_ms=0,
message=( message="调度失败策略为 stop,未再启动",
"调度失败策略为 stop,未再启动"
if stop_all
else "上游节点全部终止且至少一个失败,已跳过"
),
) )
session.add(skipped) session.add(skipped)
latest[node_id] = skipped latest[node_id] = skipped
@@ -640,26 +623,9 @@ class DispatchOrchestrator:
"skipped node: run={} node={} reason={}", "skipped node: run={} node={} reason={}",
run.run_id[-12:], run.run_id[-12:],
node_id[-12:], node_id[-12:],
"stop_policy" if stop_all else "parents_blocked", "stop_policy",
) )
elif ( elif parents_terminal and active_count < max_concurrency:
# Dispatch only when every parent has actually
# run to completion successfully. A None parent
# means the parent has not even been dispatched
# yet (e.g. upstream is still queued); the existing
# parents_blocked branch above handles the case
# where every parent is terminal but at least one
# failed.
# ``all([])`` is intentionally true: a root node has
# no parents and must be eligible for the initial
# dispatch that starts the DAG.
all(
item is not None
and item.node_status == "succeeded"
for item in parent_runs
)
and active_count < max_concurrency
):
dispatched = await self._dispatch_node( dispatched = await self._dispatch_node(
session, session,
run=run, run=run,