修复失败策略运行逻辑

This commit is contained in:
Winnie
2026-08-18 11:10:10 +08:00
parent ff700d8db4
commit 3b0229f018
+9 -43
View File
@@ -553,6 +553,7 @@ class DispatchOrchestrator:
latest[row.node_id] = row
max_concurrency = max(1, int(snapshot.get("max_concurrency", 1)))
failure_policy = snapshot.get("failure_policy", "stop")
while True:
changed = False
active_count = sum(
@@ -586,7 +587,7 @@ class DispatchOrchestrator:
for item in latest.values()
)
stop_all = (
snapshot.get("failure_policy", "stop") == "stop"
failure_policy == "stop"
and exhausted_failure
)
for node in nodes:
@@ -594,29 +595,15 @@ class DispatchOrchestrator:
if node_id in latest:
continue
parent_runs = [latest.get(parent) for parent in parents[node_id]]
# Decide whether this node should be skipped. A node
# is only skipped when we know it can never run:
# * ``stop_all`` — the whole run was aborted on the
# first failure, so any not-yet-dispatched node is
# dropped;
# * all parents are terminal AND at least one
# failed — there is no remaining success path.
# If even one parent is still ``queued`` or
# ``running`` we keep waiting: under ``failure_policy
# == 'continue'`` a sibling might still succeed and
# the failed parent does not block that.
# 只有 ``stop`` 策略才会在失败后跳过尚未启动的节点。
# ``continue`` 表示“前一个节点失败也继续往后执行”,因此
# 下游只需等待所有上游结束,不要求它们全部成功。
parents_terminal = all(
item is not None
and item.node_status in TERMINAL_NODE_STATES
for item in parent_runs
)
any_parent_failed = any(
item is not None
and item.node_status in FAILED_NODE_STATES
for item in parent_runs
)
parents_blocked = parents_terminal and any_parent_failed
if stop_all or parents_blocked:
if stop_all:
skipped = ScheduleNodeRuns(
node_run_id=new_ulid(),
run_id=run.run_id,
@@ -627,11 +614,7 @@ class DispatchOrchestrator:
state_version=1,
finished_at=utcnow(),
duration_ms=0,
message=(
"调度失败策略为 stop,未再启动"
if stop_all
else "上游节点全部终止且至少一个失败,已跳过"
),
message="调度失败策略为 stop,未再启动",
)
session.add(skipped)
latest[node_id] = skipped
@@ -640,26 +623,9 @@ class DispatchOrchestrator:
"skipped node: run={} node={} reason={}",
run.run_id[-12:],
node_id[-12:],
"stop_policy" if stop_all else "parents_blocked",
"stop_policy",
)
elif (
# Dispatch only when every parent has actually
# run to completion successfully. A None parent
# means the parent has not even been dispatched
# yet (e.g. upstream is still queued); the existing
# parents_blocked branch above handles the case
# where every parent is terminal but at least one
# failed.
# ``all([])`` is intentionally true: a root node has
# no parents and must be eligible for the initial
# dispatch that starts the DAG.
all(
item is not None
and item.node_status == "succeeded"
for item in parent_runs
)
and active_count < max_concurrency
):
elif parents_terminal and active_count < max_concurrency:
dispatched = await self._dispatch_node(
session,
run=run,