update: rel path
This commit is contained in:
@@ -1,7 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
import os
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status
|
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status
|
||||||
@@ -10,6 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
|
|
||||||
from common.db.models import DataResources, StorageObjects
|
from common.db.models import DataResources, StorageObjects
|
||||||
from common.ids import new_ulid
|
from common.ids import new_ulid
|
||||||
|
from common.storage import workspaces_root
|
||||||
from common.storage.schemas import (
|
from common.storage.schemas import (
|
||||||
CreateUploadRequest,
|
CreateUploadRequest,
|
||||||
DownloadUrlRequest,
|
DownloadUrlRequest,
|
||||||
@@ -25,6 +28,7 @@ from backend.schemas import (
|
|||||||
CreateResourceUploadRequest,
|
CreateResourceUploadRequest,
|
||||||
DownloadUrlRequest,
|
DownloadUrlRequest,
|
||||||
)
|
)
|
||||||
|
from backend.schemas import ResourceRelativePathRequest
|
||||||
from backend.services.storage import (
|
from backend.services.storage import (
|
||||||
create_download_url_payload,
|
create_download_url_payload,
|
||||||
create_server_object_payload,
|
create_server_object_payload,
|
||||||
@@ -36,23 +40,49 @@ from backend.services.storage import (
|
|||||||
router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"])
|
router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"])
|
||||||
|
|
||||||
|
|
||||||
|
def compute_jupyter_relative_path(script_path: str, resource_relative: str) -> str:
|
||||||
|
"""从当前脚本所在目录算到资源文件的 Jupyter 相对路径。
|
||||||
|
|
||||||
|
script_path / resource_relative 都是相对于 user root_dir 的 POSIX 路径。
|
||||||
|
"""
|
||||||
|
script_dir = PurePosixPath(script_path).parent.as_posix()
|
||||||
|
if not script_dir or script_dir == ".":
|
||||||
|
return resource_relative
|
||||||
|
return os.path.relpath(resource_relative, start=script_dir)
|
||||||
|
|
||||||
|
|
||||||
def resource_payload(
|
def resource_payload(
|
||||||
resource: DataResources,
|
resource: DataResources,
|
||||||
storage_object: StorageObjects,
|
storage_object: StorageObjects,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
# The Jupyter-side path: storage_object.object_key is stored as
|
# ``object_key`` is stored as ``workspace/{ws_id}/{user_id}/.resources/{file_name}``.
|
||||||
# "workspace/{ws_id}/{user_id}/{ulid}{ext}" and Jupyter's root_dir is
|
# Two Jupyter-facing path views:
|
||||||
# "/workspace/{ws_id}", so Jupyter sees the file at
|
# - ``jupyter_accessible_path``: per-user Jupyter-relative path. The
|
||||||
# "{user_id}/{ulid}{ext}". Strip the workspace/{ws_id}/ prefix so the
|
# Jupyter root_dir is already ``workspace/{ws_id}/{user_id}/``, so we
|
||||||
# frontend can render a copy-pasteable Jupyter path.
|
# only return the tail ``.resources/{file_name}`` for copy-paste use.
|
||||||
|
# - ``absolute_path``: full filesystem path inside the Jupyter
|
||||||
|
# container (the rclone mount root). Shape:
|
||||||
|
# ``{workspaces_root}/{ws_id}/{user_id}/.resources/{file_name}``.
|
||||||
workspace_prefix = f"workspace/{resource.workspace_id}/"
|
workspace_prefix = f"workspace/{resource.workspace_id}/"
|
||||||
|
user_prefix = f"{resource.owner_user_id}/"
|
||||||
|
jupyter_accessible_path = ""
|
||||||
|
absolute_path = ""
|
||||||
if storage_object.object_key and storage_object.object_key.startswith(
|
if storage_object.object_key and storage_object.object_key.startswith(
|
||||||
workspace_prefix
|
workspace_prefix
|
||||||
):
|
):
|
||||||
jupyter_accessible_path = storage_object.object_key[
|
remainder = storage_object.object_key[len(workspace_prefix):]
|
||||||
len(workspace_prefix) :
|
if remainder.startswith(user_prefix):
|
||||||
]
|
tail = remainder[len(user_prefix):] # ``.resources/{file_name}``
|
||||||
else:
|
jupyter_accessible_path = tail
|
||||||
|
absolute_path = (
|
||||||
|
workspaces_root()
|
||||||
|
/ resource.workspace_id
|
||||||
|
/ resource.owner_user_id
|
||||||
|
/ Path(tail)
|
||||||
|
).as_posix()
|
||||||
|
else:
|
||||||
|
jupyter_accessible_path = remainder
|
||||||
|
elif storage_object.object_key:
|
||||||
jupyter_accessible_path = storage_object.object_key
|
jupyter_accessible_path = storage_object.object_key
|
||||||
return {
|
return {
|
||||||
"resource_id": resource.resource_id,
|
"resource_id": resource.resource_id,
|
||||||
@@ -74,6 +104,7 @@ def resource_payload(
|
|||||||
"object_status": storage_object.object_status,
|
"object_status": storage_object.object_status,
|
||||||
},
|
},
|
||||||
"jupyter_accessible_path": jupyter_accessible_path,
|
"jupyter_accessible_path": jupyter_accessible_path,
|
||||||
|
"absolute_path": absolute_path,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -89,6 +120,39 @@ def can_view(resource: DataResources, context: RequestContext) -> bool:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{resource_id}/jupyter-relative-path")
|
||||||
|
async def resource_jupyter_relative_path(
|
||||||
|
resource_id: str,
|
||||||
|
payload: ResourceRelativePathRequest,
|
||||||
|
context: RequestContext = Depends(request_context),
|
||||||
|
session: AsyncSession = Depends(database_session),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
resource, storage_object = await get_visible_resource(
|
||||||
|
resource_id, context, session
|
||||||
|
)
|
||||||
|
script_path = payload.script_path.strip()
|
||||||
|
if (
|
||||||
|
script_path.startswith("/")
|
||||||
|
or "\\" in script_path
|
||||||
|
or any(seg == ".." for seg in script_path.split("/"))
|
||||||
|
or any(ord(c) < 0x20 or ord(c) == 0x7F for c in script_path)
|
||||||
|
):
|
||||||
|
raise HTTPException(422, "script_path must be a clean Jupyter-relative POSIX path")
|
||||||
|
|
||||||
|
resource_path = resource_payload(resource, storage_object)["jupyter_accessible_path"]
|
||||||
|
if not resource_path.startswith(".resources/"):
|
||||||
|
raise HTTPException(
|
||||||
|
status.HTTP_409_CONFLICT,
|
||||||
|
"resource is not stored under .resources/",
|
||||||
|
)
|
||||||
|
relative = compute_jupyter_relative_path(script_path, resource_path)
|
||||||
|
return {
|
||||||
|
"request_id": context.request_id,
|
||||||
|
"data": {"relative_path": relative},
|
||||||
|
"meta": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/uploads", status_code=status.HTTP_201_CREATED)
|
@router.post("/uploads", status_code=status.HTTP_201_CREATED)
|
||||||
async def create_resource_upload(
|
async def create_resource_upload(
|
||||||
payload: CreateResourceUploadRequest,
|
payload: CreateResourceUploadRequest,
|
||||||
|
|||||||
@@ -60,3 +60,7 @@ class PublishVersionRequest(StrictModel):
|
|||||||
|
|
||||||
class DownloadUrlRequest(StrictModel):
|
class DownloadUrlRequest(StrictModel):
|
||||||
expires_seconds: int = Field(default=300, ge=30, le=3600)
|
expires_seconds: int = Field(default=300, ge=30, le=3600)
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceRelativePathRequest(StrictModel):
|
||||||
|
script_path: str = Field(min_length=1, max_length=512)
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ from typing import Any
|
|||||||
from fastapi import HTTPException, Request, status
|
from fastapi import HTTPException, Request, status
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
|
||||||
from common.config import settings
|
from common.config import settings
|
||||||
from common.db.models import StorageObjects, UploadSessions
|
from common.db.models import StorageObjects, UploadSessions
|
||||||
@@ -54,7 +55,31 @@ from common.storage import build_storage_uri
|
|||||||
|
|
||||||
|
|
||||||
def _safe_file_name(value: str) -> str:
|
def _safe_file_name(value: str) -> str:
|
||||||
return value.strip() or "upload.bin"
|
"""Sanitize a user-supplied filename for use in an S3 object key.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- Strip whitespace
|
||||||
|
- Reject empty / `.` / `..` (path traversal)
|
||||||
|
- Reject leading `.` (hidden files)
|
||||||
|
- Reject `/`, `\\`, and any control character (ASCII < 0x20 or 0x7F)
|
||||||
|
- Reject leading/trailing whitespace already handled by .strip()
|
||||||
|
The Jupyter-side editor selection depends on the suffix, so callers
|
||||||
|
use ``PurePosixPath(safe).suffix`` to recover the extension.
|
||||||
|
"""
|
||||||
|
name = value.strip()
|
||||||
|
if (
|
||||||
|
not name
|
||||||
|
or name in {".", ".."}
|
||||||
|
or name.startswith(".")
|
||||||
|
or "/" in name
|
||||||
|
or "\\" in name
|
||||||
|
or any(ord(char) < 0x20 or ord(char) == 0x7F for char in name)
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||||
|
detail="invalid file_name",
|
||||||
|
)
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
def _utcnow_naive() -> Any:
|
def _utcnow_naive() -> Any:
|
||||||
@@ -167,11 +192,15 @@ async def create_upload_record(
|
|||||||
payload.usage_type,
|
payload.usage_type,
|
||||||
workspace_artifact_bucket=workspace.artifact_bucket,
|
workspace_artifact_bucket=workspace.artifact_bucket,
|
||||||
)
|
)
|
||||||
# Keep the opaque upload id while preserving the original extension.
|
# Use the sanitized file_name as the on-disk basename (no ULID). The
|
||||||
# Jupyter selects its editor from this suffix, so an extensionless
|
# extension is still derived for StorageObjects.file_extension.
|
||||||
# object would make notebooks look like generic JSON/text files.
|
safe_name = _safe_file_name(payload.file_name)
|
||||||
file_extension = PurePosixPath(_safe_file_name(payload.file_name)).suffix.lower()
|
file_extension = PurePosixPath(safe_name).suffix.lower()
|
||||||
object_key = f"{payload.workspace_id}/{payload.user_id}/{new_ulid()}{file_extension}"
|
# Use the sanitized file_name as the on-disk basename (no ULID).
|
||||||
|
# DataResources files live under a per-user ``.resources`` subdir
|
||||||
|
# so they don't collide with workspace-tree scripts/directories
|
||||||
|
# the user authors directly in Jupyter.
|
||||||
|
object_key = f"{payload.workspace_id}/{payload.user_id}/.resources/{safe_name}"
|
||||||
upload = UploadSessions(
|
upload = UploadSessions(
|
||||||
upload_id=new_ulid(),
|
upload_id=new_ulid(),
|
||||||
workspace_id=payload.workspace_id,
|
workspace_id=payload.workspace_id,
|
||||||
@@ -305,7 +334,15 @@ async def upload_bytes_to_session(
|
|||||||
usage_type=upload.usage_type,
|
usage_type=upload.usage_type,
|
||||||
)
|
)
|
||||||
session.add(item)
|
session.add(item)
|
||||||
await session.flush()
|
try:
|
||||||
|
await session.flush()
|
||||||
|
except IntegrityError as exc:
|
||||||
|
upload.upload_status = "failed"
|
||||||
|
await session.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status.HTTP_409_CONFLICT,
|
||||||
|
"a file with this name already exists at this path; rename and retry",
|
||||||
|
) from exc
|
||||||
await session.refresh(item)
|
await session.refresh(item)
|
||||||
upload.storage_object_id = item.storage_object_id
|
upload.storage_object_id = item.storage_object_id
|
||||||
upload.upload_status = "completed"
|
upload.upload_status = "completed"
|
||||||
|
|||||||
Reference in New Issue
Block a user