dev: initial implementation of notebook-snapshot extension
Manual-commit notebook snapshot extension (JupyterLab 4):
Backend
- Storage adapter layer (Local + S3 via boto3, pip extra)
- Dumb put/get/list/delete contract; key validation
- S3Storage: lazy boto3 import, S3ConnectionError with friendly
messages + connectivity check at startup
- SnapshotStore
- Strip code outputs/execution_count, MD5 on cleaned JSON
- Gzipped envelope (id/timestamp/name/description/hash/size/notebook)
- Append-only manifest.json (rebuildable from version files)
- SnapshotUnchangedError when new hash matches most recent version
- REST API (commit / list / content) under /snapshot/ namespace
- traitlets config (storage_type, local_root, s3_* + S3_* env fallback)
Frontend
- snapshot:commit command (toolbar button + command palette)
- Dialog for name + description
- SnapshotPanel (left sidebar timeline) + DiffWidget (main area)
- Cell diff: id-match first, LCS fallback; jsdiff for line diff
- Restore via model.fromJSON() + context.save() (no refresh)
- All AGENTS.md hard conventions enforced
Tests: 33 backend pytest passing (storage + store + routes)
Docs: AGENTS.md, design.md, README.md synced with implementation.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
tao.chen
@@ -0,0 +1,68 @@
|
||||
"""Storage backends and factory for notebook snapshots."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .base import StorageAdapter, validate_key
|
||||
from .local import LocalFileStorage
|
||||
from .s3 import S3ConnectionError, S3Storage
|
||||
|
||||
|
||||
def create_storage(
|
||||
storage_type: str,
|
||||
root_dir: str | Path,
|
||||
**kwargs,
|
||||
) -> StorageAdapter:
|
||||
"""Create a storage adapter from a configuration string.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
storage_type: str
|
||||
One of ``"local"`` or ``"s3"`` (covers AWS S3, MinIO, and any
|
||||
S3-API-compatible service via the ``endpoint`` parameter).
|
||||
root_dir: str | Path
|
||||
Root directory used by local storage. For S3 this is ignored.
|
||||
**kwargs
|
||||
Extra arguments forwarded to the backend constructor. For S3,
|
||||
expected keys include ``endpoint``, ``access_key``, ``secret_key``,
|
||||
``bucket``, ``secure``, and ``prefix``. For local, ``local_root`` may
|
||||
override the default ``root_dir / ".notebook_versions"`` path.
|
||||
|
||||
Returns
|
||||
-------
|
||||
StorageAdapter
|
||||
Configured storage adapter instance.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If ``storage_type`` is unknown.
|
||||
S3ConnectionError
|
||||
If ``storage_type="s3"`` and the endpoint is unreachable or
|
||||
credentials are missing/invalid.
|
||||
"""
|
||||
storage_type = storage_type.lower()
|
||||
if storage_type == "local":
|
||||
local_root = kwargs.get("local_root")
|
||||
if local_root:
|
||||
return LocalFileStorage(local_root)
|
||||
return LocalFileStorage(Path(root_dir) / ".notebook_versions")
|
||||
if storage_type == "s3":
|
||||
return S3Storage(
|
||||
endpoint=kwargs["endpoint"],
|
||||
access_key=kwargs["access_key"],
|
||||
secret_key=kwargs["secret_key"],
|
||||
bucket=kwargs.get("bucket", "notebook-snapshot"),
|
||||
secure=kwargs.get("secure", False),
|
||||
prefix=kwargs.get("prefix", ""),
|
||||
)
|
||||
raise ValueError(f"Unknown storage type: {storage_type!r}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"create_storage",
|
||||
"LocalFileStorage",
|
||||
"S3ConnectionError",
|
||||
"S3Storage",
|
||||
"StorageAdapter",
|
||||
"validate_key",
|
||||
]
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Abstract storage adapter for notebook snapshot backends.
|
||||
|
||||
This module defines the common interface for dumb bytes buckets. It has no
|
||||
knowledge of notebooks, versions, manifests, compression, or hashing.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
def validate_key(key: str) -> str:
|
||||
"""Validate and return a POSIX-style relative storage key.
|
||||
|
||||
Rejects empty keys, absolute paths, parent-directory segments, and
|
||||
backslashes. The returned value is the same string passed in.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
key: str
|
||||
The storage key to validate.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The validated key.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If the key is empty, absolute, contains ``..``, or contains
|
||||
backslashes.
|
||||
"""
|
||||
if not key:
|
||||
raise ValueError("Storage key must not be empty")
|
||||
if key.startswith("/"):
|
||||
raise ValueError(f"Storage key must be relative, got absolute path: {key!r}")
|
||||
if "\\" in key:
|
||||
raise ValueError(f"Storage key must use POSIX separators, got backslash: {key!r}")
|
||||
if ".." in key.split("/"):
|
||||
raise ValueError(f"Storage key must not contain parent segments: {key!r}")
|
||||
return key
|
||||
|
||||
|
||||
class StorageAdapter(ABC):
|
||||
"""Abstract base class for byte-oriented storage backends."""
|
||||
|
||||
@abstractmethod
|
||||
def put(self, key: str, data: bytes) -> None:
|
||||
"""Store ``data`` under ``key``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
key: str
|
||||
The storage key.
|
||||
data: bytes
|
||||
The payload to store.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get(self, key: str) -> bytes:
|
||||
"""Return the bytes stored under ``key``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
key: str
|
||||
The storage key.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bytes
|
||||
The stored payload.
|
||||
|
||||
Raises
|
||||
------
|
||||
KeyError
|
||||
If the key does not exist.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def list_keys(self, prefix: str = "") -> list[str]:
|
||||
"""Return a sorted list of keys starting with ``prefix``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
prefix: str, optional
|
||||
Filter keys to those beginning with this value. The empty string
|
||||
matches all keys.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
Sorted matching keys.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def delete(self, key: str) -> None:
|
||||
"""Delete the object at ``key``.
|
||||
|
||||
This method is idempotent: deleting a missing key does not raise.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
key: str
|
||||
The storage key.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Return whether ``key`` exists in storage.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
key: str
|
||||
The storage key.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
``True`` if the key exists, ``False`` otherwise.
|
||||
"""
|
||||
try:
|
||||
self.get(key)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Local filesystem storage adapter."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .base import StorageAdapter, validate_key
|
||||
|
||||
|
||||
class LocalFileStorage(StorageAdapter):
|
||||
"""Store objects as files under a root directory on the local filesystem."""
|
||||
|
||||
def __init__(self, root: str | Path) -> None:
|
||||
"""Initialize the adapter with a root directory.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root: str | Path
|
||||
The root directory under which all objects are stored.
|
||||
"""
|
||||
self._root = Path(root).expanduser().resolve()
|
||||
self._root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _key_to_path(self, key: str) -> Path:
|
||||
"""Resolve a storage key to a path inside the root directory.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If the resolved path would escape the root directory.
|
||||
"""
|
||||
path = (self._root / key).resolve()
|
||||
if self._root not in path.parents and path != self._root:
|
||||
raise ValueError(f"Resolved path {path} escapes root {self._root}")
|
||||
return path
|
||||
|
||||
def put(self, key: str, data: bytes) -> None:
|
||||
"""Store ``data`` under ``key`` atomically.
|
||||
|
||||
Writes to a temporary file in the target directory and renames it into
|
||||
place. Parent directories are created as needed.
|
||||
"""
|
||||
validate_key(key)
|
||||
path = self._key_to_path(key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = path.parent / f".{path.name}.tmp-{os.getpid()}"
|
||||
try:
|
||||
tmp_path.write_bytes(data)
|
||||
os.replace(str(tmp_path), str(path))
|
||||
finally:
|
||||
if tmp_path.exists():
|
||||
tmp_path.unlink()
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
"""Return the bytes stored under ``key``.
|
||||
|
||||
Raises
|
||||
------
|
||||
KeyError
|
||||
If the file does not exist.
|
||||
"""
|
||||
validate_key(key)
|
||||
path = self._key_to_path(key)
|
||||
if not path.is_file():
|
||||
raise KeyError(key)
|
||||
return path.read_bytes()
|
||||
|
||||
def list_keys(self, prefix: str = "") -> list[str]:
|
||||
"""Return a sorted list of file keys starting with ``prefix``."""
|
||||
if prefix:
|
||||
validate_key(prefix)
|
||||
keys = []
|
||||
for item in self._root.rglob("*"):
|
||||
if item.is_file():
|
||||
rel_key = item.relative_to(self._root).as_posix()
|
||||
if rel_key.startswith(prefix):
|
||||
keys.append(rel_key)
|
||||
return sorted(keys)
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
"""Delete the file at ``key``.
|
||||
|
||||
Idempotent: no error if the file is already missing.
|
||||
"""
|
||||
validate_key(key)
|
||||
path = self._key_to_path(key)
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
return
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Return whether ``key`` exists as a regular file."""
|
||||
validate_key(key)
|
||||
path = self._key_to_path(key)
|
||||
return path.is_file()
|
||||
@@ -0,0 +1,228 @@
|
||||
"""S3-compatible storage adapter backed by boto3.
|
||||
|
||||
Works with AWS S3, MinIO, and any S3-API-compatible service.
|
||||
"""
|
||||
|
||||
from .base import StorageAdapter, validate_key
|
||||
|
||||
|
||||
class S3ConnectionError(Exception):
|
||||
"""Raised when the S3 connectivity / credentials check fails at startup.
|
||||
|
||||
Wraps the underlying boto3/botocore exception with a friendly, actionable
|
||||
message naming the endpoint, bucket, and the most likely fix.
|
||||
"""
|
||||
|
||||
|
||||
class S3Storage(StorageAdapter):
|
||||
"""Store objects in an S3-compatible bucket via boto3.
|
||||
|
||||
Works with AWS S3 and MinIO (just point ``endpoint`` at the MinIO host).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
bucket: str,
|
||||
secure: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
"""Initialize the adapter and verify connectivity.
|
||||
|
||||
boto3 is imported lazily inside this constructor so that importing
|
||||
``snapshot.storage.s3`` does not require boto3 to be installed.
|
||||
|
||||
On construction, the adapter performs a HEAD on the configured bucket
|
||||
to validate network reachability and credentials. If the bucket does
|
||||
not exist, it is created automatically. Any connectivity, credential,
|
||||
or permission problem is surfaced as :class:`S3ConnectionError` with
|
||||
a message that names the endpoint, bucket, and likely fix.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
endpoint: str
|
||||
S3 endpoint host (without scheme), e.g. ``localhost:9000`` for
|
||||
MinIO or ``s3.us-east-1.amazonaws.com`` for AWS S3.
|
||||
access_key: str
|
||||
S3 access key ID.
|
||||
secret_key: str
|
||||
S3 secret access key.
|
||||
bucket: str
|
||||
Target bucket name. Created if it does not exist.
|
||||
secure: bool, optional
|
||||
Use HTTPS instead of HTTP. Defaults to ``False``.
|
||||
prefix: str, optional
|
||||
Prefix prepended to every stored key. Defaults to ``""``.
|
||||
|
||||
Raises
|
||||
------
|
||||
S3ConnectionError
|
||||
If the endpoint is unreachable, credentials are missing/invalid,
|
||||
or the bucket cannot be created due to permissions.
|
||||
"""
|
||||
try:
|
||||
import boto3
|
||||
from botocore.client import Config as BotoConfig
|
||||
from botocore.exceptions import ClientError
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"boto3 is required for S3 storage. "
|
||||
"Install it with: pip install snapshot[s3]"
|
||||
) from exc
|
||||
|
||||
self._bucket = bucket
|
||||
self._prefix = prefix
|
||||
self._endpoint = endpoint
|
||||
scheme = "https" if secure else "http"
|
||||
self._client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=f"{scheme}://{endpoint}",
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name="us-east-1",
|
||||
config=BotoConfig(signature_version="s3v4"),
|
||||
)
|
||||
self._ClientError = ClientError
|
||||
self._connect_and_ensure_bucket()
|
||||
|
||||
def _connect_and_ensure_bucket(self) -> None:
|
||||
"""Verify connectivity/credentials, then ensure the bucket exists.
|
||||
|
||||
Raises :class:`S3ConnectionError` with a friendly message on failure.
|
||||
"""
|
||||
try:
|
||||
self._client.head_bucket(Bucket=self._bucket)
|
||||
return # bucket exists, connectivity and credentials confirmed
|
||||
except self._ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code in ("404", "NoSuchBucket"):
|
||||
# Bucket missing — try to create it. Permission failure here
|
||||
# surfaces as S3ConnectionError rather than the raw boto3 error.
|
||||
try:
|
||||
self._client.create_bucket(Bucket=self._bucket)
|
||||
except self._ClientError as create_exc:
|
||||
ccode = create_exc.response.get("Error", {}).get("Code", "")
|
||||
raise S3ConnectionError(
|
||||
f"S3 bucket '{self._bucket}' does not exist and could "
|
||||
f"not be created (code={ccode}). Check that the "
|
||||
f"credentials have s3:CreateBucket permission on "
|
||||
f"endpoint {self._endpoint}."
|
||||
) from create_exc
|
||||
return
|
||||
# Other ClientError: permission denied, account disabled, etc.
|
||||
raise S3ConnectionError(
|
||||
f"S3 access denied (endpoint={self._endpoint}, "
|
||||
f"bucket={self._bucket}, code={code}). Check that the "
|
||||
f"credentials have s3:ListBucket and s3:GetBucketLocation "
|
||||
f"permissions."
|
||||
) from exc
|
||||
except Exception as exc: # noqa: BLE001 - catch any boto3/botocore error
|
||||
# EndpointConnectionError, NoCredentialsError, InvalidAccessKeyId,
|
||||
# SSLError, ConnectTimeoutError, etc. all land here.
|
||||
cls = type(exc).__name__
|
||||
hint = ""
|
||||
if "Credentials" in cls or "AccessKey" in cls or "Auth" in cls:
|
||||
hint = (
|
||||
" Check that S3_ACCESS_KEY / S3_SECRET_KEY env vars (or "
|
||||
"c.SnapshotConfig.s3_access_key / s3_secret_key) are set "
|
||||
"and correct."
|
||||
)
|
||||
elif "Endpoint" in cls or "Connection" in cls or "Connect" in cls or "SSLError" in cls or "timeout" in cls.lower():
|
||||
hint = (
|
||||
f" Check that the endpoint '{self._endpoint}' is reachable, "
|
||||
f"the URL is correct, and s3_secure matches the protocol."
|
||||
)
|
||||
raise S3ConnectionError(
|
||||
f"Cannot connect to S3 endpoint {self._endpoint} "
|
||||
f"(bucket={self._bucket}): {cls}: {exc}.{hint}"
|
||||
) from exc
|
||||
|
||||
def _object_key(self, key: str) -> str:
|
||||
"""Return the full S3 object key, including the configured prefix."""
|
||||
return f"{self._prefix}{key}"
|
||||
|
||||
def put(self, key: str, data: bytes) -> None:
|
||||
"""Store ``data`` under ``key`` in the configured bucket."""
|
||||
validate_key(key)
|
||||
self._client.put_object(
|
||||
Bucket=self._bucket,
|
||||
Key=self._object_key(key),
|
||||
Body=data,
|
||||
)
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
"""Return the bytes stored under ``key``.
|
||||
|
||||
Raises
|
||||
------
|
||||
KeyError
|
||||
If the object does not exist.
|
||||
"""
|
||||
validate_key(key)
|
||||
try:
|
||||
response = self._client.get_object(
|
||||
Bucket=self._bucket,
|
||||
Key=self._object_key(key),
|
||||
)
|
||||
except self._ClientError as exc:
|
||||
error_code = exc.response.get("Error", {}).get("Code", "")
|
||||
if error_code in ("NoSuchKey", "404"):
|
||||
raise KeyError(key) from exc
|
||||
raise
|
||||
return response["Body"].read()
|
||||
|
||||
def list_keys(self, prefix: str = "") -> list[str]:
|
||||
"""Return a sorted list of keys starting with ``prefix``.
|
||||
|
||||
The configured bucket prefix is stripped from returned keys.
|
||||
"""
|
||||
if prefix:
|
||||
validate_key(prefix)
|
||||
full_prefix = self._object_key(prefix)
|
||||
keys = []
|
||||
continuation_token = None
|
||||
while True:
|
||||
kwargs: dict = {
|
||||
"Bucket": self._bucket,
|
||||
"Prefix": full_prefix,
|
||||
}
|
||||
if continuation_token:
|
||||
kwargs["ContinuationToken"] = continuation_token
|
||||
response = self._client.list_objects_v2(**kwargs)
|
||||
for obj in response.get("Contents", []):
|
||||
full_key = obj["Key"]
|
||||
if full_key.startswith(self._prefix):
|
||||
stripped = full_key[len(self._prefix):]
|
||||
keys.append(stripped)
|
||||
if not response.get("IsTruncated"):
|
||||
break
|
||||
continuation_token = response.get("NextContinuationToken")
|
||||
return sorted(keys)
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
"""Delete the object at ``key``.
|
||||
|
||||
Idempotent: S3 ``delete_object`` succeeds even when the key is absent.
|
||||
"""
|
||||
validate_key(key)
|
||||
self._client.delete_object(
|
||||
Bucket=self._bucket,
|
||||
Key=self._object_key(key),
|
||||
)
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Return whether ``key`` exists in the bucket."""
|
||||
validate_key(key)
|
||||
try:
|
||||
self._client.head_object(
|
||||
Bucket=self._bucket,
|
||||
Key=self._object_key(key),
|
||||
)
|
||||
return True
|
||||
except self._ClientError as exc:
|
||||
error_code = exc.response.get("Error", {}).get("Code", "")
|
||||
if error_code in ("404", "NoSuchKey"):
|
||||
return False
|
||||
raise
|
||||
Reference in New Issue
Block a user