refactor: drop fetch_url body cap and modernize datetime usage

- fetch_url: remove the 1MB body cap and the streaming helper. The
  manual redirect loop with allowlist re-check (the SSRF fix) is kept
  intact; the body is now read in full via resp.text. Drop the now-
  meaningless `truncated` field from FetchUrlResult and the tests that
  asserted on it. Switched from httpx.stream() back to httpx.get()
  for the redirect loop — cleaner without the body cap.

- datetime: replace deprecated datetime.utcnow() with
  datetime.now(timezone.utc) in submit.py (3 sites) and
  core/job_writer.py (1 site). Update the stale comment in
  core/pending_store.py that referenced the old call.

- Clean up an unused `from common.config import settings` import in
  submit.py that ruff flagged.

Co-Authored-By: Claude <noreply@anthropic.com>

spark_executor/tools/fetch_url.py

@@ -21,7 +21,6 @@ from spark_executor.core.yarn_client import YarnClientConfig
 from spark_executor.models import FetchUrlResult
 from spark_executor.tools.connections import store as conn_store
 
-_MAX_BODY_BYTES = 1_000_000  # 1 MB cap on response body
 _REQUEST_TIMEOUT_SECONDS = 30
 
 _MAX_REDIRECTS = 3
@@ -70,29 +69,6 @@ def _validate_url_host(url: str, allowlist: list[str] | None) -> None:
     )
 
 
-def _read_streamed_body(resp: httpx.Response) -> tuple[bytes, bool]:
-    """Stream the response body, keeping at most ``_MAX_BODY_BYTES`` bytes.
-
-    Returns ``(body_bytes, truncated)``. Stops reading as soon as the cap is
-    exceeded so that a multi-gigabyte response from an allowlisted host cannot
-    OOM the service.
-    """
-    chunks: list[bytes] = []
-    total = 0
-    truncated = False
-    for chunk in resp.iter_bytes():
-        if total + len(chunk) <= _MAX_BODY_BYTES:
-            chunks.append(chunk)
-            total += len(chunk)
-        else:
-            if total < _MAX_BODY_BYTES:
-                chunks.append(chunk[: _MAX_BODY_BYTES - total])
-                total = _MAX_BODY_BYTES
-            truncated = True
-            break
-    return b"".join(chunks), truncated
-
-
 def fetch_url(url: str, connection_name: str) -> FetchUrlResult:
     """Proxy an HTTP GET to url using the auth/SSL settings of connection_name."""
     logger.debug(f"fetch_url enter url={url} connection_name={connection_name}")
@@ -107,45 +83,39 @@ def fetch_url(url: str, connection_name: str) -> FetchUrlResult:
     verify = config.verify_for_httpx()
 
     for hop in range(_MAX_REDIRECTS + 1):
-        with httpx.stream(
-            "GET",
+        resp = httpx.get(
             url,
             auth=auth,
             verify=verify,
             timeout=_REQUEST_TIMEOUT_SECONDS,
             follow_redirects=False,
-        ) as resp:
-            if resp.status_code not in _REDIRECT_STATUSES:
-                body_bytes, truncated = _read_streamed_body(resp)
-                encoding = resp.encoding or "utf-8"
-                body = body_bytes.decode(encoding, errors="replace")
-
-                logger.info(
-                    f"fetch_url ok url={url} connection_name={connection_name} "
-                    f"status_code={resp.status_code} "
-                    f"content_type={resp.headers.get('content-type', '')} "
-                    f"body_bytes={len(body)} truncated={truncated}"
-                )
-
-                return FetchUrlResult(
-                    url=url,
-                    status_code=resp.status_code,
-                    content_type=resp.headers.get("content-type", ""),
-                    body=body,
-                    truncated=truncated,
-                )
-
-            location = resp.headers.get("Location") or resp.headers.get("location")
-            if not location:
-                raise ValueError(
-                    f"HTTP GET returned {resp.status_code} with no Location header at {url}"
-                )
-            next_url = urljoin(url, location)
-            logger.debug(
-                f"fetch_url redirect url={url} status={resp.status_code} to={next_url}"
+        )
+        if resp.status_code not in _REDIRECT_STATUSES:
+            logger.info(
+                f"fetch_url ok url={url} connection_name={connection_name} "
+                f"status_code={resp.status_code} "
+                f"content_type={resp.headers.get('content-type', '')} "
+                f"body_chars={len(resp.text)}"
             )
-            _validate_url_host(next_url, conn.url_allowlist)
-            url = next_url
+
+            return FetchUrlResult(
+                url=url,
+                status_code=resp.status_code,
+                content_type=resp.headers.get("content-type", ""),
+                body=resp.text,
+            )
+
+        location = resp.headers.get("Location") or resp.headers.get("location")
+        if not location:
+            raise ValueError(
+                f"HTTP GET returned {resp.status_code} with no Location header at {url}"
+            )
+        next_url = urljoin(url, location)
+        logger.debug(
+            f"fetch_url redirect url={url} status={resp.status_code} to={next_url}"
+        )
+        _validate_url_host(next_url, conn.url_allowlist)
+        url = next_url
 
     raise ValueError(
         f"fetch_url exceeded {_MAX_REDIRECTS} redirects (last url: {url})"