Replace the old RustFS-specific storage layer (common.storage.client /
RustFSObjectStore) with a minimal sync/async abstraction:
AsyncStorageBackend: put / get / get_stream / delete / exists / stat /
list / get_url / copy
StorageBackend: same surface, sync implementations
create_storage({"type": "s3" | "local", "mode": "async", ...})
backends/s3.py: S3-compatible (boto3 / aioboto3)
backends/local.py: on-disk filesystem (aiofiles)
Concretely:
- Drop RustFSObjectStore + common.storage.client (deleted).
- Drop the RustFS-specific ensure_bucket / presign_put / move_to_trash /
rewrite_to_public_path / sha256 / put_bytes methods.
- Migrate backend/storage_api.py + backend/main.py + backend/scripts.py
+ schedule/service.py + schedule/worker.py to the new abstraction.
- Migrate backend/storage_client.py + schedule/storage_client.py to
stub status (HTTP wrapper is dead code post-migration; rewrite pending).
- Rename all RUSTFS_* env vars to S3_* across .env.example,
docker-compose.yml, default.conf, scripts/nginx-entrypoint.sh,
common/config.py.
- Replace hardcoded rclone remote name "rustfs" with "s3" in
docker-compose.yml + config.py default.
- Rename "rustfs" SQLAlchemy column comments + table comments to
provider-neutral wording; StorageObjects.storage_backend enum
value moves from "rustfs" to "s3" (DB rows with the old value will
fail the != "s3" check until a one-shot migration is applied).
- Drop unused common/src/common/migrations/{README,env.py,script.py.mako}
(the alembic setup lives in /migrations/, not here).
Migration of the old abstractions has been done in one pass; per-route
method calls (delete / stat / put / get_url) are now direct one-liners
against AsyncStorageBackend.
After this commit:
- All Python imports resolve; routes compile (compileall green).
- s3 mode is fully wired.
- Routes that depended on removed methods (presign_put, move_to_trash,
rewrite_to_public_path, head() metadata) raise NotImplementedError
with a one-line TODO; rewriting these route handlers is the next step.
161 lines
6.6 KiB
Plaintext
161 lines
6.6 KiB
Plaintext
# ----------------------------------------------------------------------------
|
||
# NOTE: this file is mounted into the nginx container as a TEMPLATE.
|
||
# scripts/nginx-entrypoint.sh (mounted as /docker-entrypoint.sh) substitutes
|
||
# the single ${S3_ENDPOINT} placeholder at container start. The rendered
|
||
# output is written to /etc/nginx/conf.d/default.conf and execs nginx.
|
||
# ----------------------------------------------------------------------------
|
||
|
||
map $http_upgrade $connection_upgrade {
|
||
default upgrade;
|
||
'' close;
|
||
}
|
||
|
||
server {
|
||
listen 80;
|
||
server_name localhost;
|
||
# 全局或针对存储服务设置最大上传限制 (100G)
|
||
client_max_body_size 100G;
|
||
# 指定 Docker 内置 DNS 解析器,并设置 30 秒缓存
|
||
resolver 127.0.0.11 valid=30s ipv6=off;
|
||
|
||
# S3 upstream — full URL passed through to proxy_pass below.
|
||
set $s3_backend "${S3_ENDPOINT}";
|
||
|
||
location / {
|
||
root /usr/share/nginx/html; # 前端静态文件存放在容器中的路径
|
||
index index.html index.htm;
|
||
|
||
# 核心:支持 SPA 路由(前端 React Router / Vue Router 刷新不报 404)
|
||
try_files $uri $uri/ /index.html;
|
||
}
|
||
|
||
#(可选)针对静态文件资源加长期缓存优化
|
||
# location ~* \.(?:css|js|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm|htc|woff|woff2)$ {
|
||
# root /usr/share/nginx/html;
|
||
# expires 2h; # 前端静态缓存2小时
|
||
# add_header Cache-Control "public, no-transform";
|
||
# }
|
||
|
||
location /api/ {
|
||
proxy_pass http://backend:8000/api/;
|
||
proxy_set_header Host $host;
|
||
proxy_set_header X-Forwarded-Host $http_host;
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
proxy_set_header X-Real-IP $remote_addr;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
# Explicitly forward the session cookie set by
|
||
# POST /api/v1/auth/login. nginx forwards it by default, but
|
||
# spelling it out keeps the auth contract visible.
|
||
proxy_set_header Cookie $http_cookie;
|
||
# Defense in depth: blank out the legacy identity headers so a
|
||
# malicious client cannot bypass the cookie-based auth flow
|
||
# by stuffing X-User-ID / X-Workspace-ID into the request.
|
||
# The backend's RequestContext no longer reads them (it
|
||
# derives identity from the access_token cookie), so this is
|
||
# belt-and-suspenders against a future regression.
|
||
proxy_set_header X-User-ID "";
|
||
proxy_set_header X-Workspace-ID "";
|
||
}
|
||
|
||
# =========================================================================
|
||
# 1. S3 对象存储服务转发 (/storage/)
|
||
# =========================================================================
|
||
location /storage/ {
|
||
# 核心:透传 Host,确保 S3 生成的 Presigned URL 包含公网地址
|
||
proxy_set_header Host $http_host;
|
||
proxy_set_header X-Real-IP $remote_addr;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
|
||
# 转发至 S3($s3_backend 来自 set 指令;尾斜杠保留 location /storage/ 前缀剥离语义)
|
||
proxy_pass $s3_backend/;
|
||
|
||
# HTTP/1.1 长连接支持
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Connection "";
|
||
|
||
# 性能优化:关闭双向 Buffer,实现大文件流式传输
|
||
proxy_buffering off;
|
||
proxy_request_buffering off;
|
||
|
||
# 大文件传输超时设置
|
||
proxy_connect_timeout 300s;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
}
|
||
|
||
# =========================================================================
|
||
# 2. Jupyter 服务配置
|
||
# =========================================================================
|
||
# 拒绝直接访问目录
|
||
location ~ ^/jupyter/(?<workspace_id>[^/]+)/?$ {
|
||
return 403 "Direct directory access is forbidden. Please specify a notebook path.";
|
||
}
|
||
|
||
# 精准匹配入口:仅允许访问特定 notebook 页面与配套资源
|
||
location ~ ^/jupyter/(?<workspace_id>[^/]+)(?<rest_uri>/.*)$ {
|
||
|
||
# 触发后端鉴权
|
||
auth_request /internal-auth;
|
||
|
||
auth_request_set $target_upstream $upstream_http_x_upstream_addr;
|
||
auth_request_set $jupyter_token $upstream_http_x_jupyter_internal_token;
|
||
|
||
# 拼接完整 URL 并输出到 Response Header 方便调试
|
||
add_header X-Debug-Full-Url "$target_upstream/jupyter/$workspace_id$rest_uri$is_args$args" always;
|
||
|
||
# 代理到具体的 Jupyter 子进程
|
||
proxy_pass $target_upstream/jupyter/$workspace_id$rest_uri$is_args$args;
|
||
proxy_set_header Authorization "token $jupyter_token";
|
||
|
||
# 支持 WebSocket (Jupyter Kernel 必需)
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Upgrade $http_upgrade;
|
||
proxy_set_header Connection $connection_upgrade;
|
||
|
||
proxy_set_header Host $host;
|
||
proxy_set_header X-Real-IP $remote_addr;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
# Defense in depth: do not let the browser-supplied identity
|
||
# headers leak past the auth subrequest. The auth subrequest
|
||
# only forwards the Cookie + Authorization it cares about;
|
||
# the actual Jupyter upstream is fully trusted (the address
|
||
# comes from the backend's runtime registry), so a leaked
|
||
# X-User-ID here would not matter for the proxy target but
|
||
# could pollute audit logs.
|
||
proxy_set_header X-User-ID "";
|
||
proxy_set_header X-Workspace-ID "";
|
||
}
|
||
|
||
# 2. 内部 Auth 子请求 location
|
||
location = /internal-auth {
|
||
internal;
|
||
# 打到宿主机的 FastAPI 8000 端口
|
||
proxy_pass http://backend:8000/api/v1/auth/jupyter;
|
||
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
|
||
proxy_pass_header x-upstream-addr;
|
||
proxy_pass_header x-jupyter-internal-token;
|
||
|
||
proxy_set_header X-Original-Workspace-Id $workspace_id;
|
||
proxy_set_header X-Original-URI $request_uri;
|
||
|
||
proxy_set_header Cookie $http_cookie;
|
||
proxy_set_header Authorization $http_authorization;
|
||
# Defense in depth: the auth subrequest reads the session
|
||
# cookie / Authorization header, not the legacy identity
|
||
# headers. Blank them out so a poisoned client header cannot
|
||
# be confused for an authenticated identity if the backend
|
||
# code is ever refactored to read them again.
|
||
proxy_set_header X-User-ID "";
|
||
proxy_set_header X-Workspace-ID "";
|
||
}
|
||
|
||
# 拒绝其余非法路径
|
||
location /jupyter/ {
|
||
return 403 "Access Denied";
|
||
}
|
||
}
|