refactor: drop fetch_url body cap and modernize datetime usage
- fetch_url: remove the 1MB body cap and the streaming helper. The manual redirect loop with allowlist re-check (the SSRF fix) is kept intact; the body is now read in full via resp.text. Drop the now- meaningless `truncated` field from FetchUrlResult and the tests that asserted on it. Switched from httpx.stream() back to httpx.get() for the redirect loop — cleaner without the body cap. - datetime: replace deprecated datetime.utcnow() with datetime.now(timezone.utc) in submit.py (3 sites) and core/job_writer.py (1 site). Update the stale comment in core/pending_store.py that referenced the old call. - Clean up an unused `from common.config import settings` import in submit.py that ruff flagged. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -6,9 +6,8 @@
|
||||
import os
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from common.config import settings
|
||||
from common.logging import logger
|
||||
from common.sql_guard import validate_pyspark_code
|
||||
from spark_executor.core.connection_store import store as conn_store
|
||||
@@ -124,7 +123,7 @@ def prepare_submit_job(
|
||||
num_executors=num_executors,
|
||||
spark_conf=dict(conn.spark_conf),
|
||||
extra_args=dict(extra_args or {}),
|
||||
created_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc),
|
||||
status="PENDING",
|
||||
)
|
||||
pending_store.save(pending)
|
||||
@@ -248,7 +247,7 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
|
||||
application_id=application_id,
|
||||
script_path=pending.script_path,
|
||||
queue=pending.queue,
|
||||
submit_time=datetime.utcnow(),
|
||||
submit_time=datetime.now(timezone.utc),
|
||||
connection=pending.connection,
|
||||
yarn_rm_url=pending.yarn_rm_url,
|
||||
)
|
||||
@@ -282,7 +281,7 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
|
||||
application_id=application_id,
|
||||
script_path=pending.script_path,
|
||||
queue=pending.queue,
|
||||
submit_time=datetime.utcnow(),
|
||||
submit_time=datetime.now(timezone.utc),
|
||||
connection=pending.connection,
|
||||
yarn_rm_url=pending.yarn_rm_url,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user