feat: schedule node add python version

This commit is contained in:
tao.chen
2026-08-11 15:56:05 +08:00
parent e28bc39328
commit fe7f1a744e
9 changed files with 128 additions and 21 deletions
+3 -1
View File
@@ -5,7 +5,7 @@ from typing import Any
from pydantic import Field, field_validator, model_validator from pydantic import Field, field_validator, model_validator
from common.db.models.schedules import FailurePolicy, TriggerType from common.db.models.schedules import FailurePolicy, PythonVersion, TriggerType
from common.schemas import StrictModel from common.schemas import StrictModel
@@ -114,6 +114,7 @@ class CreateScheduleNodeRequest(StrictModel):
position_y: float = Field(default=0, ge=-100_000, le=100_000) position_y: float = Field(default=0, ge=-100_000, le=100_000)
arguments_json: dict[str, Any] = Field(default_factory=dict, max_length=100) arguments_json: dict[str, Any] = Field(default_factory=dict, max_length=100)
env_refs_json: dict[str, str] = Field(default_factory=dict, max_length=100) env_refs_json: dict[str, str] = Field(default_factory=dict, max_length=100)
python_version: PythonVersion = Field(default="3.12")
@field_validator("node_key", "node_name") @field_validator("node_key", "node_name")
@classmethod @classmethod
@@ -146,6 +147,7 @@ class UpdateScheduleNodeRequest(StrictModel):
default=None, default=None,
max_length=100, max_length=100,
) )
python_version: PythonVersion | None = Field(default=None)
@field_validator("node_name") @field_validator("node_name")
@classmethod @classmethod
+3
View File
@@ -151,6 +151,7 @@ def node_payload(
"node_key": item.node_key, "node_key": item.node_key,
"node_name": item.node_name, "node_name": item.node_name,
"versions_id": item.versions_id, "versions_id": item.versions_id,
"python_version": item.python_version,
"timeout_seconds": item.timeout_seconds, "timeout_seconds": item.timeout_seconds,
"retry_count": item.retry_count, "retry_count": item.retry_count,
"retry_interval_sec": item.retry_interval_sec, "retry_interval_sec": item.retry_interval_sec,
@@ -787,6 +788,7 @@ async def create_schedule_node(
node_key=payload.node_key, node_key=payload.node_key,
node_name=payload.node_name, node_name=payload.node_name,
versions_id=payload.versions_id, versions_id=payload.versions_id,
python_version=payload.python_version,
timeout_seconds=payload.timeout_seconds, timeout_seconds=payload.timeout_seconds,
retry_count=payload.retry_count, retry_count=payload.retry_count,
retry_interval_sec=payload.retry_interval_sec, retry_interval_sec=payload.retry_interval_sec,
@@ -837,6 +839,7 @@ async def update_schedule_node(
mutable_fields = { mutable_fields = {
"node_name", "node_name",
"versions_id", "versions_id",
"python_version",
"timeout_seconds", "timeout_seconds",
"retry_count", "retry_count",
"retry_interval_sec", "retry_interval_sec",
+5
View File
@@ -11,6 +11,7 @@ from common.db.base import Base
TriggerType = Literal["manual", "cron", "api"] TriggerType = Literal["manual", "cron", "api"]
FailurePolicy = Literal["stop", "continue"] FailurePolicy = Literal["stop", "continue"]
PythonVersion = Literal["3.8", "3.10", "3.12"]
class Schedules(Base): class Schedules(Base):
@@ -140,6 +141,10 @@ class ScheduleNodes(Base):
) )
node_name: Mapped[str] = mapped_column(String(255), nullable=False) node_name: Mapped[str] = mapped_column(String(255), nullable=False)
versions_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) versions_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
python_version: Mapped[str] = mapped_column(
String(8), nullable=False, server_default=text("'3.12'"),
comment="节点执行 Python 版本(3.8/3.10/3.12",
)
timeout_seconds: Mapped[int] = mapped_column( timeout_seconds: Mapped[int] = mapped_column(
INTEGER, nullable=False, server_default=text("600") INTEGER, nullable=False, server_default=text("600")
) )
@@ -179,6 +179,7 @@ def upgrade() -> None:
sa.Column('node_key', sa.String(length=64), nullable=False, comment='画布内稳定标识'), sa.Column('node_key', sa.String(length=64), nullable=False, comment='画布内稳定标识'),
sa.Column('node_name', sa.String(length=255), nullable=False), sa.Column('node_name', sa.String(length=255), nullable=False),
sa.Column('versions_id', mysql.CHAR(length=26), nullable=False), sa.Column('versions_id', mysql.CHAR(length=26), nullable=False),
sa.Column('python_version', mysql.String(length=8), nullable=False, server_default=sa.text("'3.12'"), comment='节点执行 Python 版本(3.8/3.10/3.12'),
sa.Column('timeout_seconds', mysql.INTEGER(), server_default=sa.text('600'), nullable=False), sa.Column('timeout_seconds', mysql.INTEGER(), server_default=sa.text('600'), nullable=False),
sa.Column('retry_count', mysql.INTEGER(), server_default=sa.text('0'), nullable=False), sa.Column('retry_count', mysql.INTEGER(), server_default=sa.text('0'), nullable=False),
sa.Column('retry_interval_sec', mysql.INTEGER(), server_default=sa.text('5'), nullable=False), sa.Column('retry_interval_sec', mysql.INTEGER(), server_default=sa.text('5'), nullable=False),
@@ -0,0 +1,34 @@
"""add schedule_nodes.python_version
Revision ID: g7h8i9j0k1l2
Revises: f6a7b8c9d0e1
Create Date: 2026-08-11
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
revision: str = "g7h8i9j0k1l2"
down_revision: str | Sequence[str] | None = "f6a7b8c9d0e1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.add_column(
"schedule_nodes",
sa.Column(
"python_version",
sa.String(length=8),
nullable=False,
server_default=sa.text("'3.12'"),
comment="节点执行 Python 版本(3.8/3.10/3.12",
),
)
def downgrade() -> None:
op.drop_column("schedule_nodes", "python_version")
+20 -5
View File
@@ -7,13 +7,28 @@ COPY extras ./extras
RUN UV_DEFAULT_INDEX="https://pypi.tuna.tsinghua.edu.cn/simple/" uv sync --frozen --no-dev --no-editable --package schedule RUN UV_DEFAULT_INDEX="https://pypi.tuna.tsinghua.edu.cn/simple/" uv sync --frozen --no-dev --no-editable --package schedule
RUN uv venv --python 3.12 /opt/venv/python3.12 && \ RUN uv venv --python 3.12 /opt/venv/python3.12 && \
UV_DEFAULT_INDEX="https://pypi.tuna.tsinghua.edu.cn/simple/" uv pip install --python /opt/venv/python3.12/bin/python -r extras/requirements-py312 UV_DEFAULT_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple/ \
uv pip install --python /opt/venv/python3.12/bin/python -r extras/requirements-py312 && \
RUN uv venv --python 3.8 /opt/venv/python3.8 && \ /opt/venv/python3.12/bin/python -m ipykernel install \
UV_DEFAULT_INDEX="https://pypi.tuna.tsinghua.edu.cn/simple/" uv pip install --python /opt/venv/python3.8/bin/python -r extras/requirements-py38 --prefix=/usr/local \
--name=python312 \
--display-name="Python 3.12"
RUN uv venv --python 3.10 /opt/venv/python3.10 && \ RUN uv venv --python 3.10 /opt/venv/python3.10 && \
UV_DEFAULT_INDEX="https://pypi.tuna.tsinghua.edu.cn/simple/" uv pip install --python /opt/venv/python3.10/bin/python -r extras/requirements-py310 UV_DEFAULT_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple/ \
uv pip install --python /opt/venv/python3.10/bin/python -r extras/requirements-py310 && \
/opt/venv/python3.10/bin/python -m ipykernel install \
--prefix=/usr/local \
--name=python310 \
--display-name="Python 3.10"
RUN uv venv --python 3.8 /opt/venv/python3.8 && \
UV_DEFAULT_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple/ \
uv pip install --python /opt/venv/python3.8/bin/python -r extras/requirements-py38 && \
/opt/venv/python3.8/bin/python -m ipykernel install \
--prefix=/usr/local \
--name=python38 \
--display-name="Python 3.8"
EXPOSE 8000 EXPOSE 8000
CMD ["uv", "run", "--frozen", "--package", "schedule", "gunicorn", "--config", "schedule/gunicorn.conf.py", "schedule.main:app"] CMD ["uv", "run", "--frozen", "--package", "schedule", "gunicorn", "--config", "schedule/gunicorn.conf.py", "schedule.main:app"]
+21 -6
View File
@@ -39,12 +39,14 @@ async def _execute_notebook(
artifact_name: str, artifact_name: str,
arguments: list[str], arguments: list[str],
timeout_seconds: int, timeout_seconds: int,
python_version: str = "3.12",
) -> ExecutionResult: ) -> ExecutionResult:
output = artifact.with_name(f"executed-{artifact_name}") output = artifact.with_name(f"executed-{artifact_name}")
logger.debug( logger.debug(
"notebook exec start: artifact={} timeout={}s args={}", "notebook exec start: artifact={} timeout={}s version={} args={}",
artifact_name, artifact_name,
timeout_seconds, timeout_seconds,
python_version,
len(arguments), len(arguments),
) )
process = await asyncio.create_subprocess_exec( process = await asyncio.create_subprocess_exec(
@@ -59,6 +61,8 @@ async def _execute_notebook(
str(max(1, timeout_seconds)), str(max(1, timeout_seconds)),
"--arguments-json", "--arguments-json",
json.dumps(arguments, ensure_ascii=False), json.dumps(arguments, ensure_ascii=False),
"--python-version",
python_version,
cwd=str(artifact.parent), cwd=str(artifact.parent),
stdout=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT, stderr=asyncio.subprocess.STDOUT,
@@ -76,9 +80,10 @@ async def _execute_notebook(
error_code = "NODE_TIMEOUT" error_code = "NODE_TIMEOUT"
error_message = f"notebook exceeded timeout of {timeout_seconds} seconds" error_message = f"notebook exceeded timeout of {timeout_seconds} seconds"
logger.warning( logger.warning(
"notebook exec timed out: artifact={} timeout={}s", "notebook exec timed out: artifact={} timeout={}s version={}",
artifact_name, artifact_name,
timeout_seconds, timeout_seconds,
python_version,
) )
else: else:
exit_code = process.returncode exit_code = process.returncode
@@ -114,9 +119,10 @@ async def _execute_notebook(
error_message=error_message, error_message=error_message,
) )
logger.info( logger.info(
"notebook exec done: artifact={} status={} exit_code={}", "notebook exec done: artifact={} status={} version={} exit_code={}",
artifact_name, artifact_name,
result.status, result.status,
python_version,
result.exit_code, result.exit_code,
) )
return result return result
@@ -127,11 +133,15 @@ async def _execute_python(
*, *,
arguments: list[str], arguments: list[str],
timeout_seconds: int, timeout_seconds: int,
python_version: str = "3.12",
) -> ExecutionResult: ) -> ExecutionResult:
# NOTE: The worker process runs a single interpreter, so .py scripts
# continue to use sys.executable. Switching interpreters is future work.
logger.debug( logger.debug(
"python exec start: artifact={} timeout={}s args={}", "python exec start: artifact={} timeout={}s version={} args={}",
artifact.name, artifact.name,
timeout_seconds, timeout_seconds,
python_version,
len(arguments), len(arguments),
) )
process = await asyncio.create_subprocess_exec( process = await asyncio.create_subprocess_exec(
@@ -156,9 +166,10 @@ async def _execute_python(
ensure_ascii=False, ensure_ascii=False,
).encode("utf-8") ).encode("utf-8")
logger.warning( logger.warning(
"python exec timed out: artifact={} timeout={}s", "python exec timed out: artifact={} timeout={}s version={}",
artifact.name, artifact.name,
timeout_seconds, timeout_seconds,
python_version,
) )
return ExecutionResult( return ExecutionResult(
status="timed_out", status="timed_out",
@@ -190,9 +201,10 @@ async def _execute_python(
error_message=message, error_message=message,
) )
logger.info( logger.info(
"python exec done: artifact={} status={} exit_code={}", "python exec done: artifact={} status={} version={} exit_code={}",
artifact.name, artifact.name,
result.status, result.status,
python_version,
result.exit_code, result.exit_code,
) )
return result return result
@@ -207,6 +219,7 @@ async def execute_artifact(
artifact_path: str, artifact_path: str,
arguments: list[str], arguments: list[str],
timeout_seconds: int, timeout_seconds: int,
python_version: str = "3.12",
) -> ExecutionResult: ) -> ExecutionResult:
logger.debug( logger.debug(
"execute_artifact: run={} node={} script_type={}", "execute_artifact: run={} node={} script_type={}",
@@ -230,12 +243,14 @@ async def execute_artifact(
artifact_name=artifact_name, artifact_name=artifact_name,
arguments=arguments, arguments=arguments,
timeout_seconds=timeout_seconds, timeout_seconds=timeout_seconds,
python_version=python_version,
) )
if script_type == "python": if script_type == "python":
return await _execute_python( return await _execute_python(
artifact, artifact,
arguments=arguments, arguments=arguments,
timeout_seconds=timeout_seconds, timeout_seconds=timeout_seconds,
python_version=python_version,
) )
logger.error("unsupported script_type: {}", script_type) logger.error("unsupported script_type: {}", script_type)
raise ValueError(f"unsupported script_type: {script_type}") raise ValueError(f"unsupported script_type: {script_type}")
+13 -9
View File
@@ -1,14 +1,11 @@
from __future__ import annotations
import argparse import argparse
import json import json
import sys import sys
import traceback import traceback
from pathlib import Path from pathlib import Path
from loguru import logger
import nbformat import nbformat
from loguru import logger
from nbclient import NotebookClient from nbclient import NotebookClient
@@ -27,8 +24,7 @@ def emit_outputs(notebook: object) -> None:
) )
elif output_type == "error": elif output_type == "error":
print( print(
f"{output.get('ename', 'Error')}: " f"{output.get('ename', 'Error')}: {output.get('evalue', '')}",
f"{output.get('evalue', '')}",
file=sys.stderr, file=sys.stderr,
flush=True, flush=True,
) )
@@ -39,6 +35,11 @@ def main() -> None:
parser.add_argument("--input", required=True) parser.add_argument("--input", required=True)
parser.add_argument("--output", required=True) parser.add_argument("--output", required=True)
parser.add_argument("--timeout", required=True, type=int) parser.add_argument("--timeout", required=True, type=int)
parser.add_argument(
"--python-version",
choices=("3.8", "3.10", "3.12"),
default="3.12",
)
parser.add_argument("--arguments-json", default="[]") parser.add_argument("--arguments-json", default="[]")
args = parser.parse_args() args = parser.parse_args()
@@ -51,9 +52,10 @@ def main() -> None:
raise ValueError("arguments-json must contain an array of strings") raise ValueError("arguments-json must contain an array of strings")
logger.info( logger.info(
"notebook runner start: input={} timeout={}s args={}", "notebook runner start: input={} timeout={}s python={} args={}",
source.name, source.name,
args.timeout, args.timeout,
args.python_version,
len(arguments), len(arguments),
) )
notebook = nbformat.read(source, as_version=4) notebook = nbformat.read(source, as_version=4)
@@ -68,14 +70,16 @@ def main() -> None:
) )
exit_code = 0 exit_code = 0
try: try:
kernel_name = f"python{args.python_version.replace('.', '')}"
client = NotebookClient( client = NotebookClient(
notebook, notebook,
timeout=max(1, args.timeout), timeout=max(1, args.timeout),
kernel_name="python3", kernel_name=kernel_name,
allow_errors=False, allow_errors=False,
) )
logger.debug( logger.debug(
"notebook client created: kernel=python3 timeout={}s", "notebook client created: kernel={} timeout={}s",
kernel_name,
max(1, args.timeout), max(1, args.timeout),
) )
# No explicit cwd — the kernel inherits the parent's cwd, which the # No explicit cwd — the kernel inherits the parent's cwd, which the
+28
View File
@@ -28,6 +28,7 @@ from common.db import session_scope
from common.db.models import ( from common.db.models import (
ConsumerInbox, ConsumerInbox,
ScheduleNodeRuns, ScheduleNodeRuns,
ScheduleNodes,
ScheduleRuns, ScheduleRuns,
Schedules, Schedules,
StorageObjects, StorageObjects,
@@ -86,6 +87,9 @@ class NodeExecutor:
object_key=context["object_key"], object_key=context["object_key"],
content_hash=context["content_hash"], content_hash=context["content_hash"],
) )
python_version = await self._node_python_version(
payload["node_run_id"]
)
result = await execute_artifact( result = await execute_artifact(
content, content,
run_id=payload["run_id"], run_id=payload["run_id"],
@@ -94,6 +98,7 @@ class NodeExecutor:
artifact_path=payload["artifact_path"], artifact_path=payload["artifact_path"],
arguments=[str(item) for item in payload.get("arguments", [])], arguments=[str(item) for item in payload.get("arguments", [])],
timeout_seconds=int(payload["timeout_seconds"]), timeout_seconds=int(payload["timeout_seconds"]),
python_version=python_version,
) )
except Exception as exc: except Exception as exc:
trace = traceback.format_exc() trace = traceback.format_exc()
@@ -250,6 +255,29 @@ class NodeExecutor:
) )
return True return True
async def _node_python_version(
self,
node_run_id: str,
) -> str:
async with self.session_factory() as session:
row = (
await session.execute(
select(ScheduleNodes.python_version)
.join(
ScheduleNodeRuns,
ScheduleNodeRuns.node_id == ScheduleNodes.node_id,
)
.where(ScheduleNodeRuns.node_run_id == node_run_id)
)
).one_or_none()
if row is None:
logger.warning(
"node python_version not found, defaulting to 3.12: node_run={}",
node_run_id[-12:],
)
return "3.12"
return row[0]
async def _execution_context( async def _execution_context(
self, self,
payload: dict[str, Any], payload: dict[str, Any],