fix: stabilize cron scheduling and run history
This commit is contained in:
@@ -16,7 +16,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
@@ -66,6 +66,12 @@ class CronScheduler:
|
||||
self._on_trigger = on_trigger
|
||||
_ACTIVE_TRIGGER = on_trigger
|
||||
self._sync_task: asyncio.Task[None] | None = None
|
||||
# 仅在调度配置实际变化时才重置 APScheduler job。若每 5 秒都
|
||||
# reschedule,一旦恰好落在整分钟之后,就可能把本分钟的触发跳过。
|
||||
self._job_signatures: dict[str, tuple[str, str, int]] = {}
|
||||
# APScheduler 的定时唤醒异常时,由 5 秒同步循环兜底。键保存的是
|
||||
# 已由兜底路径处理的“本地整分钟”,避免同一分钟重复提交。
|
||||
self._fallback_dispatched_minutes: dict[str, datetime] = {}
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start APScheduler and spawn the periodic sync loop."""
|
||||
@@ -120,8 +126,9 @@ class CronScheduler:
|
||||
|
||||
- adds jobs for enabled cron schedules present in MySQL
|
||||
- removes jobs whose schedule has been disabled / soft-deleted
|
||||
- updates ``Schedules.next_run_at`` from the next APScheduler tick
|
||||
- updates ``Schedules.next_run_at`` from the Cron expression itself
|
||||
"""
|
||||
due_schedule_ids: list[str] = []
|
||||
async with session_scope(self.session_factory) as session:
|
||||
schedules = list(
|
||||
(
|
||||
@@ -142,14 +149,16 @@ class CronScheduler:
|
||||
job_id = f"schedule:{item.schedule_id}"
|
||||
active_job_ids.add(job_id)
|
||||
expression = (item.cron_expression or "").strip()
|
||||
max_instances = max(1, item.max_concurrency)
|
||||
signature = (expression, item.timezone, max_instances)
|
||||
trigger = CronTrigger.from_crontab(
|
||||
expression,
|
||||
timezone=ZoneInfo(item.timezone),
|
||||
)
|
||||
if self.scheduler.get_job(job_id) is not None:
|
||||
self.scheduler.reschedule_job(job_id, trigger=trigger)
|
||||
updated_count += 1
|
||||
else:
|
||||
now = datetime.now(ZoneInfo(item.timezone))
|
||||
minute = now.replace(second=0, microsecond=0)
|
||||
job_changed = False
|
||||
if self.scheduler.get_job(job_id) is None:
|
||||
self.scheduler.add_job(
|
||||
dispatch_persisted_cron,
|
||||
trigger=trigger,
|
||||
@@ -157,12 +166,45 @@ class CronScheduler:
|
||||
id=job_id,
|
||||
replace_existing=True,
|
||||
coalesce=True,
|
||||
max_instances=max(1, item.max_concurrency),
|
||||
max_instances=max_instances,
|
||||
misfire_grace_time=60,
|
||||
)
|
||||
added_count += 1
|
||||
job = self.scheduler.get_job(job_id)
|
||||
item.next_run_at = naive_utc(job.next_run_time)
|
||||
job_changed = True
|
||||
elif self._job_signatures.get(job_id) != signature:
|
||||
# 服务重启后的首次同步也会走这里,确保持久化 job 与
|
||||
# 数据库当前配置一致;之后配置不变时保留原定时点。
|
||||
self.scheduler.reschedule_job(job_id, trigger=trigger)
|
||||
self.scheduler.modify_job(
|
||||
job_id,
|
||||
max_instances=max_instances,
|
||||
)
|
||||
updated_count += 1
|
||||
job_changed = True
|
||||
self._job_signatures[job_id] = signature
|
||||
# 以 CronTrigger 本身计算下次执行时间,不依赖 APScheduler 的
|
||||
# 内部唤醒状态;页面展示的「下次执行」也因此保持准确。
|
||||
item.next_run_at = naive_utc(
|
||||
trigger.get_next_fire_time(None, now)
|
||||
)
|
||||
|
||||
# 首次观察或刚修改表达式时,从下一个整分钟才开始兜底,符合
|
||||
# Cron 的常规语义,避免用户在本分钟中途保存后立刻多跑一次。
|
||||
if job_changed or job_id not in self._fallback_dispatched_minutes:
|
||||
self._fallback_dispatched_minutes[job_id] = minute
|
||||
|
||||
# 正常情况下 APScheduler 会在整分钟回调。实测其偶发漏唤醒时,
|
||||
# 这里每 5 秒检查一次当前分钟是否命中表达式,并补发一次。
|
||||
due_at = trigger.get_next_fire_time(
|
||||
minute - timedelta(minutes=1),
|
||||
minute,
|
||||
)
|
||||
if (
|
||||
due_at == minute
|
||||
and self._fallback_dispatched_minutes.get(job_id) != minute
|
||||
):
|
||||
self._fallback_dispatched_minutes[job_id] = minute
|
||||
due_schedule_ids.append(item.schedule_id)
|
||||
removed_count = 0
|
||||
for job in self.scheduler.get_jobs():
|
||||
if (
|
||||
@@ -170,6 +212,8 @@ class CronScheduler:
|
||||
and job.id not in active_job_ids
|
||||
):
|
||||
self.scheduler.remove_job(job.id)
|
||||
self._job_signatures.pop(job.id, None)
|
||||
self._fallback_dispatched_minutes.pop(job.id, None)
|
||||
removed_count += 1
|
||||
if added_count or updated_count or removed_count:
|
||||
logger.info(
|
||||
@@ -178,6 +222,11 @@ class CronScheduler:
|
||||
updated_count,
|
||||
removed_count,
|
||||
)
|
||||
# 在数据库同步事务提交后再创建运行记录,避免两个会话同时读取调度方案
|
||||
# 时发生不必要的锁等待。重复回调由运行记录的幂等键自动去重。
|
||||
for schedule_id in due_schedule_ids:
|
||||
logger.debug("cron fallback dispatch: schedule={}", schedule_id[-12:])
|
||||
await self._on_trigger(schedule_id)
|
||||
|
||||
|
||||
__all__ = ["CronScheduler"]
|
||||
|
||||
Reference in New Issue
Block a user