fix: stabilize cron scheduling and run history

This commit is contained in:
Winnie
2026-08-17 18:55:31 +08:00
parent 23823325e6
commit ff700d8db4
16 changed files with 320 additions and 55 deletions
+58 -9
View File
@@ -16,7 +16,7 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from datetime import UTC
from datetime import UTC, datetime, timedelta
from zoneinfo import ZoneInfo
from apscheduler.schedulers.asyncio import AsyncIOScheduler
@@ -66,6 +66,12 @@ class CronScheduler:
self._on_trigger = on_trigger
_ACTIVE_TRIGGER = on_trigger
self._sync_task: asyncio.Task[None] | None = None
# 仅在调度配置实际变化时才重置 APScheduler job。若每 5 秒都
# reschedule,一旦恰好落在整分钟之后,就可能把本分钟的触发跳过。
self._job_signatures: dict[str, tuple[str, str, int]] = {}
# APScheduler 的定时唤醒异常时,由 5 秒同步循环兜底。键保存的是
# 已由兜底路径处理的“本地整分钟”,避免同一分钟重复提交。
self._fallback_dispatched_minutes: dict[str, datetime] = {}
def start(self) -> None:
"""Start APScheduler and spawn the periodic sync loop."""
@@ -120,8 +126,9 @@ class CronScheduler:
- adds jobs for enabled cron schedules present in MySQL
- removes jobs whose schedule has been disabled / soft-deleted
- updates ``Schedules.next_run_at`` from the next APScheduler tick
- updates ``Schedules.next_run_at`` from the Cron expression itself
"""
due_schedule_ids: list[str] = []
async with session_scope(self.session_factory) as session:
schedules = list(
(
@@ -142,14 +149,16 @@ class CronScheduler:
job_id = f"schedule:{item.schedule_id}"
active_job_ids.add(job_id)
expression = (item.cron_expression or "").strip()
max_instances = max(1, item.max_concurrency)
signature = (expression, item.timezone, max_instances)
trigger = CronTrigger.from_crontab(
expression,
timezone=ZoneInfo(item.timezone),
)
if self.scheduler.get_job(job_id) is not None:
self.scheduler.reschedule_job(job_id, trigger=trigger)
updated_count += 1
else:
now = datetime.now(ZoneInfo(item.timezone))
minute = now.replace(second=0, microsecond=0)
job_changed = False
if self.scheduler.get_job(job_id) is None:
self.scheduler.add_job(
dispatch_persisted_cron,
trigger=trigger,
@@ -157,12 +166,45 @@ class CronScheduler:
id=job_id,
replace_existing=True,
coalesce=True,
max_instances=max(1, item.max_concurrency),
max_instances=max_instances,
misfire_grace_time=60,
)
added_count += 1
job = self.scheduler.get_job(job_id)
item.next_run_at = naive_utc(job.next_run_time)
job_changed = True
elif self._job_signatures.get(job_id) != signature:
# 服务重启后的首次同步也会走这里,确保持久化 job 与
# 数据库当前配置一致;之后配置不变时保留原定时点。
self.scheduler.reschedule_job(job_id, trigger=trigger)
self.scheduler.modify_job(
job_id,
max_instances=max_instances,
)
updated_count += 1
job_changed = True
self._job_signatures[job_id] = signature
# 以 CronTrigger 本身计算下次执行时间,不依赖 APScheduler 的
# 内部唤醒状态;页面展示的「下次执行」也因此保持准确。
item.next_run_at = naive_utc(
trigger.get_next_fire_time(None, now)
)
# 首次观察或刚修改表达式时,从下一个整分钟才开始兜底,符合
# Cron 的常规语义,避免用户在本分钟中途保存后立刻多跑一次。
if job_changed or job_id not in self._fallback_dispatched_minutes:
self._fallback_dispatched_minutes[job_id] = minute
# 正常情况下 APScheduler 会在整分钟回调。实测其偶发漏唤醒时,
# 这里每 5 秒检查一次当前分钟是否命中表达式,并补发一次。
due_at = trigger.get_next_fire_time(
minute - timedelta(minutes=1),
minute,
)
if (
due_at == minute
and self._fallback_dispatched_minutes.get(job_id) != minute
):
self._fallback_dispatched_minutes[job_id] = minute
due_schedule_ids.append(item.schedule_id)
removed_count = 0
for job in self.scheduler.get_jobs():
if (
@@ -170,6 +212,8 @@ class CronScheduler:
and job.id not in active_job_ids
):
self.scheduler.remove_job(job.id)
self._job_signatures.pop(job.id, None)
self._fallback_dispatched_minutes.pop(job.id, None)
removed_count += 1
if added_count or updated_count or removed_count:
logger.info(
@@ -178,6 +222,11 @@ class CronScheduler:
updated_count,
removed_count,
)
# 在数据库同步事务提交后再创建运行记录,避免两个会话同时读取调度方案
# 时发生不必要的锁等待。重复回调由运行记录的幂等键自动去重。
for schedule_id in due_schedule_ids:
logger.debug("cron fallback dispatch: schedule={}", schedule_id[-12:])
await self._on_trigger(schedule_id)
__all__ = ["CronScheduler"]