tao.chen/notebook-snapshot-extension
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

dev: initial implementation of notebook-snapshot extension

Browse Source 
Manual-commit notebook snapshot extension (JupyterLab 4):

Backend
- Storage adapter layer (Local + S3 via boto3, pip extra)
  - Dumb put/get/list/delete contract; key validation
  - S3Storage: lazy boto3 import, S3ConnectionError with friendly
    messages + connectivity check at startup
- SnapshotStore
  - Strip code outputs/execution_count, MD5 on cleaned JSON
  - Gzipped envelope (id/timestamp/name/description/hash/size/notebook)
  - Append-only manifest.json (rebuildable from version files)
  - SnapshotUnchangedError when new hash matches most recent version
- REST API (commit / list / content) under /snapshot/ namespace
- traitlets config (storage_type, local_root, s3_* + S3_* env fallback)

Frontend
- snapshot:commit command (toolbar button + command palette)
- Dialog for name + description
- SnapshotPanel (left sidebar timeline) + DiffWidget (main area)
- Cell diff: id-match first, LCS fallback; jsdiff for line diff
- Restore via model.fromJSON() + context.save() (no refresh)
- All AGENTS.md hard conventions enforced

Tests: 33 backend pytest passing (storage + store + routes)

Docs: AGENTS.md, design.md, README.md synced with implementation.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
tao.chen
2026-07-21 17:33:55 +08:00
co-authored by Claude
commit cd1aba6698
67 changed files with 17926 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
snapshot/tests/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Python unit tests for snapshot."""
snapshot/tests/test_routes.py
+17
View File
@@ -0,0 +1,17 @@
import json
async def test_hello(jp_fetch):
# When
response = await jp_fetch("snapshot", "hello")
# Then
assert response.code == 200
payload = json.loads(response.body)
assert payload == {
"data": (
"Hello, world!"
" This is the '/snapshot/hello' endpoint."
" Try visiting me in your browser!"
),
}
snapshot/tests/test_routes_api.py
+152
View File
@@ -0,0 +1,152 @@
"""Integration tests for the snapshot REST API (commit / list / content)."""
import json
import pytest
SAMPLE_NB = {
"cells": [
{
"cell_type": "code",
"id": "c1",
"source": "print('hi')",
"outputs": [{"output_type": "stream", "text": "hi"}],
"execution_count": 1,
"metadata": {},
},
{
"cell_type": "markdown",
"id": "c2",
"source": "# title",
"metadata": {},
},
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5,
}
def _post_commit(jp_fetch, path: str, name: str, description: str = ""):
body = json.dumps(
{
"path": path,
"content": SAMPLE_NB,
"name": name,
"description": description,
}
).encode()
return jp_fetch(
"snapshot",
"notebook-version",
"commit",
method="POST",
body=body,
)
async def test_commit_creates_entry(jp_fetch):
response = await _post_commit(jp_fetch, "api_a.ipynb", "v1", "first")
assert response.code == 200
payload = json.loads(response.body)
assert payload["entry"]["name"] == "v1"
assert payload["entry"]["description"] == "first"
assert payload["entry"]["id"].startswith("v_")
assert payload["entry"]["size"] > 0
assert payload["entry"]["hash"]
async def test_list_contains_committed_entry(jp_fetch):
await _post_commit(jp_fetch, "api_b.ipynb", "v1", "desc")
response = await jp_fetch(
"snapshot",
"notebook-version",
"list",
params={"path": "api_b.ipynb"},
)
assert response.code == 200
payload = json.loads(response.body)
assert len(payload["versions"]) == 1
assert payload["versions"][0]["name"] == "v1"
assert payload["versions"][0]["description"] == "desc"
async def test_content_returns_stripped_notebook(jp_fetch):
commit_resp = await _post_commit(jp_fetch, "api_c.ipynb", "v1")
entry = json.loads(commit_resp.body)["entry"]
response = await jp_fetch(
"snapshot",
"notebook-version",
"content",
params={"path": "api_c.ipynb", "id": entry["id"]},
)
assert response.code == 200
notebook = json.loads(response.body)
code = next(c for c in notebook["cells"] if c["cell_type"] == "code")
assert code["outputs"] == []
assert code["execution_count"] is None
assert notebook["nbformat"] == 4
async def test_commit_missing_name_returns_400(jp_fetch):
body = json.dumps(
{"path": "api_d.ipynb", "content": SAMPLE_NB}
).encode()
response = await jp_fetch(
"snapshot",
"notebook-version",
"commit",
method="POST",
body=body,
raise_error=False,
)
assert response.code == 400
async def test_list_unknown_path_returns_empty(jp_fetch):
response = await jp_fetch(
"snapshot",
"notebook-version",
"list",
params={"path": "never_committed_unique.ipynb"},
)
assert response.code == 200
payload = json.loads(response.body)
assert payload["versions"] == []
async def test_commit_duplicate_returns_skipped(jp_fetch):
"""Second commit with identical content must NOT save; response is {skipped: True}."""
r1 = await _post_commit(jp_fetch, "api_dup.ipynb", "v1")
assert r1.code == 200
first = json.loads(r1.body)
assert first.get("entry") is not None
assert first.get("skipped") in (None, False)
r2 = await _post_commit(jp_fetch, "api_dup.ipynb", "v2")
assert r2.code == 200 # not an error — just a warning
second = json.loads(r2.body)
assert second.get("skipped") is True
assert second.get("reason") == "unchanged"
assert "未变更" in second.get("message", "") or "unchanged" in second.get("message", "").lower()
assert second.get("entry") is None
# And the manifest still has exactly one entry.
list_resp = await jp_fetch(
"snapshot",
"notebook-version",
"list",
params={"path": "api_dup.ipynb"},
)
assert json.loads(list_resp.body)["versions"].__len__() == 1
async def test_content_unknown_id_returns_404(jp_fetch):
response = await jp_fetch(
"snapshot",
"notebook-version",
"content",
params={"path": "api_e.ipynb", "id": "v_0_doesnotexist"},
raise_error=False,
)
assert response.code == 404
snapshot/tests/test_storage_local.py
+98
View File
@@ -0,0 +1,98 @@
"""Tests for the local filesystem storage adapter."""
import pytest
from snapshot.storage import LocalFileStorage, create_storage, validate_key
from snapshot.storage.base import StorageAdapter
def _contract(adapter: StorageAdapter) -> None:
"""Run the shared storage contract against an adapter instance."""
adapter.put("a/b/file1.txt", b"hello")
adapter.put("a/b/file2.txt", b"world")
adapter.put("c/file3.txt", b"!")
assert adapter.get("a/b/file1.txt") == b"hello"
assert adapter.exists("a/b/file1.txt")
assert not adapter.exists("a/b/missing.txt")
with pytest.raises(KeyError):
adapter.get("a/b/missing.txt")
assert adapter.list_keys() == [
"a/b/file1.txt",
"a/b/file2.txt",
"c/file3.txt",
]
assert adapter.list_keys("a/b") == [
"a/b/file1.txt",
"a/b/file2.txt",
]
adapter.delete("a/b/file1.txt")
adapter.delete("a/b/file1.txt") # idempotent
assert not adapter.exists("a/b/file1.txt")
assert adapter.list_keys() == [
"a/b/file2.txt",
"c/file3.txt",
]
def test_contract(tmp_path):
"""Validate the adapter contract on a temporary directory."""
adapter = LocalFileStorage(tmp_path)
_contract(adapter)
def test_put_creates_parent_directories(tmp_path):
"""put() must create intermediate directories."""
adapter = LocalFileStorage(tmp_path)
adapter.put("deep/nested/key.bin", b"data")
assert (tmp_path / "deep" / "nested" / "key.bin").read_bytes() == b"data"
def test_atomic_put_leaves_no_tmp_files(tmp_path):
"""Atomic writes must not leak temporary files."""
adapter = LocalFileStorage(tmp_path)
adapter.put("x/y/z.txt", b"payload")
tmp_files = list(tmp_path.rglob("*.tmp-*"))
assert tmp_files == []
def test_validate_key_rejects_bad_keys():
"""validate_key must reject unsafe key strings."""
with pytest.raises(ValueError):
validate_key("")
with pytest.raises(ValueError):
validate_key("/absolute/path")
with pytest.raises(ValueError):
validate_key("../escape")
with pytest.raises(ValueError):
validate_key("a\\b")
def test_get_missing_raises_key_error(tmp_path):
"""Reading a missing key must raise KeyError."""
adapter = LocalFileStorage(tmp_path)
with pytest.raises(KeyError):
adapter.get("missing.bin")
def test_create_storage_custom_local_root(tmp_path):
"""create_storage must honor an explicit local_root directory."""
custom_dir = tmp_path / "custom_versions"
adapter = create_storage(
"local",
root_dir=tmp_path,
local_root=str(custom_dir),
)
adapter.put("notebook.ipynb", b"data")
assert (custom_dir / "notebook.ipynb").read_bytes() == b"data"
def test_create_storage_default_local_root(tmp_path):
"""create_storage must fall back to root_dir/.notebook_versions."""
adapter = create_storage("local", root_dir=tmp_path)
adapter.put("notebook.ipynb", b"data")
expected = tmp_path / ".notebook_versions" / "notebook.ipynb"
assert expected.read_bytes() == b"data"
snapshot/tests/test_storage_s3.py
+250
View File
@@ -0,0 +1,250 @@
"""Tests for the S3 storage adapter using a monkey-patched boto3."""
import pytest
boto3 = pytest.importorskip("boto3")
from botocore.exceptions import ClientError # noqa: E402
from snapshot.storage import S3ConnectionError, S3Storage
from snapshot.storage.base import StorageAdapter
class _FakeS3Client:
"""In-memory S3 client sufficient for exercising S3Storage."""
def __init__(self, **kwargs):
self.kwargs = kwargs
self._objects = {}
self._buckets = set()
# Failure injection: if set, these are raised instead of the normal
# behavior. Used to exercise the S3ConnectionError classification.
self.head_bucket_error: Exception | None = None
self.create_bucket_error: Exception | None = None
def head_bucket(self, Bucket):
if self.head_bucket_error is not None:
raise self.head_bucket_error
if Bucket not in self._buckets:
raise ClientError({"Error": {"Code": "404"}}, "HeadBucket")
def create_bucket(self, Bucket):
if self.create_bucket_error is not None:
raise self.create_bucket_error
self._buckets.add(Bucket)
def put_object(self, Bucket, Key, Body):
self._buckets.add(Bucket)
if isinstance(Body, bytes):
data = Body
else:
data = Body.read()
self._objects[(Bucket, Key)] = data
def get_object(self, Bucket, Key):
self.head_bucket(Bucket)
if (Bucket, Key) not in self._objects:
raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject")
return {"Body": _Body(self._objects[(Bucket, Key)])}
def delete_object(self, Bucket, Key):
self._objects.pop((Bucket, Key), None)
def head_object(self, Bucket, Key):
self.head_bucket(Bucket)
if (Bucket, Key) not in self._objects:
raise ClientError({"Error": {"Code": "404"}}, "HeadObject")
def list_objects_v2(self, Bucket, Prefix="", ContinuationToken=None):
self.head_bucket(Bucket)
keys = sorted(
key for (b, key) in self._objects if b == Bucket and key.startswith(Prefix)
)
return {"Contents": [{"Key": key} for key in keys], "IsTruncated": False}
class _Body:
def __init__(self, data: bytes):
self._data = data
def read(self):
return self._data
def _make_adapter(monkeypatch, prefix=""):
monkeypatch.setattr(boto3, "client", lambda *args, **kwargs: _FakeS3Client(**kwargs))
return S3Storage(
endpoint="localhost:9000",
access_key="access",
secret_key="secret",
bucket="test-bucket",
prefix=prefix,
)
def _contract(adapter: StorageAdapter) -> None:
adapter.put("a/b/file1.txt", b"hello")
adapter.put("a/b/file2.txt", b"world")
adapter.put("c/file3.txt", b"!")
assert adapter.get("a/b/file1.txt") == b"hello"
assert adapter.exists("a/b/file1.txt")
assert not adapter.exists("a/b/missing.txt")
with pytest.raises(KeyError):
adapter.get("a/b/missing.txt")
assert adapter.list_keys() == [
"a/b/file1.txt",
"a/b/file2.txt",
"c/file3.txt",
]
assert adapter.list_keys("a/b") == [
"a/b/file1.txt",
"a/b/file2.txt",
]
adapter.delete("a/b/file1.txt")
adapter.delete("a/b/file1.txt") # idempotent
assert not adapter.exists("a/b/file1.txt")
assert adapter.list_keys() == [
"a/b/file2.txt",
"c/file3.txt",
]
def test_contract(monkeypatch):
"""Validate the adapter contract against the fake S3 client."""
adapter = _make_adapter(monkeypatch)
_contract(adapter)
def test_prefix_stripping(monkeypatch):
"""list_keys must strip the configured prefix from returned keys."""
adapter = _make_adapter(monkeypatch, prefix="notebooks/")
adapter.put("version1.ipynb", b"v1")
adapter.put("version2.ipynb", b"v2")
assert adapter.list_keys() == ["version1.ipynb", "version2.ipynb"]
assert adapter.get("version1.ipynb") == b"v1"
def test_bucket_auto_creation(monkeypatch):
"""The bucket must be created automatically if it does not exist."""
adapter = _make_adapter(monkeypatch)
adapter.put("first.bin", b"x")
assert adapter.exists("first.bin")
def test_imports_work_without_boto3():
"""Module-level imports must succeed even when boto3 is unavailable."""
import snapshot
import snapshot.storage
import snapshot.storage.s3
assert snapshot.storage.s3.S3Storage is not None
assert snapshot.storage.StorageAdapter is not None
def test_lazy_import_error(monkeypatch):
"""Instantiation must raise ImportError when boto3 is absent."""
real_import = __builtins__.__import__ if hasattr(__builtins__, "__import__") else __builtins__["__import__"]
def fake_import(name, *args, **kwargs):
if name == "boto3" or name.startswith("boto3."):
raise ImportError(f"No module named {name!r}")
return real_import(name, *args, **kwargs)
monkeypatch.setattr("builtins.__import__", fake_import)
with pytest.raises(ImportError):
S3Storage(
endpoint="localhost:9000",
access_key="a",
secret_key="s",
bucket="b",
)
# ---- connectivity / credentials checks at startup ------------------------
def _make_with_fake(monkeypatch, fake: _FakeS3Client) -> S3Storage:
monkeypatch.setattr(boto3, "client", lambda *a, **kw: fake)
return S3Storage(
endpoint="localhost:9000",
access_key="access",
secret_key="secret",
bucket="test-bucket",
)
def test_permission_denied_raises_s3_connection_error(monkeypatch):
"""head_bucket returning 403/AccessDenied must surface as S3ConnectionError."""
fake = _FakeS3Client()
fake.head_bucket_error = ClientError(
{"Error": {"Code": "AccessDenied"}}, "HeadBucket"
)
with pytest.raises(S3ConnectionError) as excinfo:
_make_with_fake(monkeypatch, fake)
msg = str(excinfo.value)
assert "AccessDenied" in msg
assert "localhost:9000" in msg
assert "test-bucket" in msg
assert "s3:ListBucket" in msg or "permission" in msg.lower()
def test_create_bucket_failure_raises_s3_connection_error(monkeypatch):
"""Bucket missing AND create_bucket failing must surface as S3ConnectionError."""
fake = _FakeS3Client()
fake.create_bucket_error = ClientError(
{"Error": {"Code": "AccessDenied"}}, "CreateBucket"
)
with pytest.raises(S3ConnectionError) as excinfo:
_make_with_fake(monkeypatch, fake)
msg = str(excinfo.value)
assert "does not exist" in msg
assert "could not be created" in msg
assert "CreateBucket" in msg
def test_endpoint_unreachable_raises_s3_connection_error(monkeypatch):
"""Network failures (e.g. EndpointConnectionError) surface as S3ConnectionError."""
from botocore.exceptions import EndpointConnectionError
fake = _FakeS3Client()
fake.head_bucket_error = EndpointConnectionError(
endpoint_url="http://localhost:9000"
)
with pytest.raises(S3ConnectionError) as excinfo:
_make_with_fake(monkeypatch, fake)
msg = str(excinfo.value)
assert "Cannot connect" in msg
assert "localhost:9000" in msg
assert "EndpointConnectionError" in msg
def test_credentials_error_surfaces_credentials_hint(monkeypatch):
"""Credential-related boto3 errors mention the S3_* env vars in the hint."""
from botocore.exceptions import NoCredentialsError
fake = _FakeS3Client()
fake.head_bucket_error = NoCredentialsError()
with pytest.raises(S3ConnectionError) as excinfo:
_make_with_fake(monkeypatch, fake)
msg = str(excinfo.value)
assert "S3_ACCESS_KEY" in msg
assert "S3_SECRET_KEY" in msg
def test_connectivity_succeeds_silently_when_bucket_exists(monkeypatch):
"""If head_bucket succeeds (bucket exists), no create is called."""
fake = _FakeS3Client()
fake._buckets.add("test-bucket") # simulate pre-existing bucket
create_called = {"flag": False}
original_create = fake.create_bucket
def tracking_create(Bucket):
create_called["flag"] = True
return original_create(Bucket)
fake.create_bucket = tracking_create
_make_with_fake(monkeypatch, fake)
assert create_called["flag"] is False, "create_bucket must not be called when bucket already exists"
snapshot/tests/test_store.py
+187
View File
@@ -0,0 +1,187 @@
"""Unit tests for the SnapshotStore (manual-commit model)."""
import pytest
from snapshot.storage import create_storage
from snapshot.store import SnapshotStore
def make_store(tmp_path):
"""Create a SnapshotStore backed by a fresh local adapter under tmp_path."""
storage = create_storage(
"local",
root_dir=tmp_path,
local_root=str(tmp_path / "snaps"),
)
return SnapshotStore(storage)
SAMPLE_NB = {
"cells": [
{
"cell_type": "code",
"id": "c1",
"source": "print('hi')",
"outputs": [{"output_type": "stream", "text": "hi"}],
"execution_count": 1,
"metadata": {},
},
{
"cell_type": "markdown",
"id": "c2",
"source": "# title",
"metadata": {},
},
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5,
}
def test_sanitize_strips_code_outputs_and_exec(tmp_path):
store = make_store(tmp_path)
entry = store.commit("nb.ipynb", SAMPLE_NB, name="v1")
nb = store.get_notebook("nb.ipynb", entry["id"])
code = next(c for c in nb["cells"] if c["cell_type"] == "code")
assert code["outputs"] == []
assert code["execution_count"] is None
md = next(c for c in nb["cells"] if c["cell_type"] == "markdown")
assert md["source"] == "# title"
assert md["id"] == "c2"
def test_hash_stable_regardless_of_outputs(tmp_path):
store = make_store(tmp_path)
nb_a = {
"cells": [
{
"cell_type": "code",
"id": "c1",
"source": "x = 1",
"outputs": [{"output_type": "stream", "text": "1"}],
"execution_count": 1,
"metadata": {},
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5,
}
nb_b = {
"cells": [
{
"cell_type": "code",
"id": "c1",
"source": "x = 1",
"outputs": [
{"output_type": "display_data", "data": {"text/plain": "9999"}}
],
"execution_count": 42,
"metadata": {},
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5,
}
e1 = store.commit("hash_a.ipynb", nb_a, name="a")
e2 = store.commit("hash_b.ipynb", nb_b, name="b")
assert e1["hash"] == e2["hash"]
def test_commit_list_content_roundtrip(tmp_path):
store = make_store(tmp_path)
entry = store.commit("rt.ipynb", SAMPLE_NB, name="first", description="hello")
assert entry["name"] == "first"
assert entry["description"] == "hello"
assert entry["id"].startswith("v_")
versions = store.list_versions("rt.ipynb")
assert len(versions) == 1
assert versions[0]["id"] == entry["id"]
assert versions[0]["name"] == "first"
assert versions[0]["description"] == "hello"
nb = store.get_notebook("rt.ipynb", entry["id"])
assert nb["nbformat"] == 4
code = next(c for c in nb["cells"] if c["cell_type"] == "code")
assert code["outputs"] == []
def test_duplicate_content_skipped(tmp_path):
store = make_store(tmp_path)
e1 = store.commit("dup.ipynb", SAMPLE_NB, name="first")
# Second commit with the same content should be skipped, not saved.
import pytest as _pytest
from snapshot.store import SnapshotUnchangedError
with _pytest.raises(SnapshotUnchangedError):
store.commit("dup.ipynb", SAMPLE_NB, name="second")
versions = store.list_versions("dup.ipynb")
assert len(versions) == 1
assert versions[0]["id"] == e1["id"]
assert versions[0]["name"] == "first"
def test_edited_content_still_saves(tmp_path):
store = make_store(tmp_path)
e1 = store.commit("edit.ipynb", SAMPLE_NB, name="v1")
edited = {
"cells": [
{
"cell_type": "code",
"id": "c1",
"source": "x = 2", # different source → different hash
"outputs": [],
"execution_count": None,
"metadata": {},
},
{
"cell_type": "markdown",
"id": "c2",
"source": "# title",
"metadata": {},
},
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5,
}
e2 = store.commit("edit.ipynb", edited, name="v2")
assert e1["id"] != e2["id"]
assert e1["hash"] != e2["hash"]
assert len(store.list_versions("edit.ipynb")) == 2
def test_first_commit_never_skipped(tmp_path):
"""No prior version exists — must always save regardless of content."""
store = make_store(tmp_path)
entry = store.commit("first.ipynb", SAMPLE_NB, name="only")
assert entry["id"].startswith("v_")
assert len(store.list_versions("first.ipynb")) == 1
def test_manifest_rebuild_from_files(tmp_path):
store = make_store(tmp_path)
entry = store.commit("rebuild.ipynb", SAMPLE_NB, name="only")
# Tamper: delete manifest directly via the storage adapter
from snapshot.storage import validate_key
manifest_key = "rebuild.ipynb/manifest.json"
validate_key(manifest_key)
store._storage.delete(manifest_key)
versions = store.list_versions("rebuild.ipynb")
assert len(versions) == 1
assert versions[0]["id"] == entry["id"]
assert versions[0]["name"] == "only"
# Manifest should have been rewritten by the rebuild
rebuilt = store.list_versions("rebuild.ipynb")
assert len(rebuilt) == 1
def test_get_notebook_missing_raises_key_error(tmp_path):
store = make_store(tmp_path)
with pytest.raises(KeyError):
store.get_notebook("missing.ipynb", "v_0_doesnotexist")