uploads: dual-write uploads to DB, startup backfill, and audit log

Store now indexes every Save into the upload_files table via SetRepo. DB failures are logged as warnings and do not fail the upload because .meta.json remains the source of truth and startup backfill recovers. Add Store.Backfill to walk Root at startup and insert index rows for any pre-existing .meta.json sidecars, swallowing duplicate-key races. The upload_file MCP Tool now writes an audit_log entry on success. Tests cover dual-write args, repo-error non-failure, and backfill skipping existing rows. Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-14 11:30:39 +08:00
parent 1796b5b872
commit 876f1c9edb
7 changed files with 316 additions and 17 deletions
@@ -5,12 +5,15 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"regexp"
 	"testing"
 	"time"
+
+	"spark-mcp-go/internal/storage"
 )

 func TestStore_SaveAndRetrieve(t *testing.T) {
@@ -351,3 +354,180 @@ func TestStore_Sweep_DeletesDataWithCorruptSidecar(t *testing.T) {
 		t.Errorf("corrupt sidecar still exists")
 	}
 }
+
+type fakeUploadRepo struct {
+	calls []storage.UploadMeta
+	err   error
+}
+
+func (f *fakeUploadRepo) Create(ctx context.Context, fileID, name string, size int64, sha256Hex string, uploadedAt time.Time) error {
+	f.calls = append(f.calls, storage.UploadMeta{
+		FileID: fileID, Name: name, Size: size, SHA256: sha256Hex, UploadedAt: uploadedAt,
+	})
+	return f.err
+}
+
+func (f *fakeUploadRepo) Get(ctx context.Context, fileID string) (storage.UploadMeta, error) {
+	for _, m := range f.calls {
+		if m.FileID == fileID {
+			return m, nil
+		}
+	}
+	return storage.UploadMeta{}, storage.ErrNotFound
+}
+
+func (f *fakeUploadRepo) List(ctx context.Context, search string, limit int) ([]storage.UploadMeta, error) {
+	return f.calls, nil
+}
+
+func (f *fakeUploadRepo) Delete(ctx context.Context, fileID string) error {
+	return nil
+}
+
+func TestStore_Save_DualWrite(t *testing.T) {
+	store, err := New(t.TempDir())
+	if err != nil {
+		t.Fatalf("new store: %v", err)
+	}
+	repo := &fakeUploadRepo{}
+	store.SetRepo(repo)
+
+	data := []byte("dual-write test")
+	fileID, _, size, sha256Hex, absPath, err := store.Save(data, "dual.txt")
+	if err != nil {
+		t.Fatalf("save: %v", err)
+	}
+	if absPath == "" {
+		t.Fatal("absPath empty")
+	}
+	if len(repo.calls) != 1 {
+		t.Fatalf("repo.Create calls: got %d, want 1", len(repo.calls))
+	}
+	call := repo.calls[0]
+	if call.FileID != fileID {
+		t.Errorf("FileID: got %q, want %q", call.FileID, fileID)
+	}
+	if call.Name != "dual.txt" {
+		t.Errorf("Name: got %q, want dual.txt", call.Name)
+	}
+	if call.Size != size {
+		t.Errorf("Size: got %d, want %d", call.Size, size)
+	}
+	if call.SHA256 != sha256Hex {
+		t.Errorf("SHA256: got %q, want %q", call.SHA256, sha256Hex)
+	}
+	if call.UploadedAt.IsZero() {
+		t.Errorf("UploadedAt is zero")
+	}
+}
+
+func TestStore_Save_RepoErrorDoesNotFailUpload(t *testing.T) {
+	store, err := New(t.TempDir())
+	if err != nil {
+		t.Fatalf("new store: %v", err)
+	}
+	repo := &fakeUploadRepo{err: errors.New("db down")}
+	store.SetRepo(repo)
+
+	data := []byte("db error test")
+	fileID, _, _, _, absPath, err := store.Save(data, "error.txt")
+	if err != nil {
+		t.Fatalf("save should not fail when repo errors: %v", err)
+	}
+	if _, err := os.Stat(absPath); err != nil {
+		t.Errorf("data file missing: %v", err)
+	}
+	if _, err := os.Stat(absPath + ".meta.json"); err != nil {
+		t.Errorf("sidecar missing: %v", err)
+	}
+	if len(repo.calls) != 1 {
+		t.Errorf("repo.Create calls: got %d, want 1", len(repo.calls))
+	}
+	if repo.calls[0].FileID != fileID {
+		t.Errorf("repo call file_id: got %q, want %q", repo.calls[0].FileID, fileID)
+	}
+}
+
+func TestStore_Backfill(t *testing.T) {
+	store, err := New(t.TempDir())
+	if err != nil {
+		t.Fatalf("new store: %v", err)
+	}
+	repo := &fakeUploadRepo{}
+
+	now := time.Now()
+	ids := []string{
+		"00000000000000000000000000000001",
+		"00000000000000000000000000000002",
+		"00000000000000000000000000000003",
+	}
+	for i, id := range ids {
+		sc := map[string]any{
+			"name":        fmt.Sprintf("file%d.txt", i),
+			"size":        i + 1,
+			"sha256":      fmt.Sprintf("sha%d", i),
+			"uploaded_at": now.Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano),
+		}
+		b, _ := json.Marshal(sc)
+		if err := os.WriteFile(filepath.Join(store.Root, id+".meta.json"), b, 0o600); err != nil {
+			t.Fatalf("create sidecar %s: %v", id, err)
+		}
+	}
+
+	inserted, err := store.Backfill(context.Background(), repo)
+	if err != nil {
+		t.Fatalf("backfill: %v", err)
+	}
+	if inserted != 3 {
+		t.Errorf("inserted=%d, want 3", inserted)
+	}
+	if len(repo.calls) != 3 {
+		t.Errorf("repo.Create calls: got %d, want 3", len(repo.calls))
+	}
+
+	// Second backfill must skip existing rows.
+	inserted, err = store.Backfill(context.Background(), repo)
+	if err != nil {
+		t.Fatalf("second backfill: %v", err)
+	}
+	if inserted != 0 {
+		t.Errorf("second inserted=%d, want 0", inserted)
+	}
+	if len(repo.calls) != 3 {
+		t.Errorf("repo.Create calls after second backfill: got %d, want 3", len(repo.calls))
+	}
+}
+
+func TestStore_Backfill_SkipsExisting(t *testing.T) {
+	store, err := New(t.TempDir())
+	if err != nil {
+		t.Fatalf("new store: %v", err)
+	}
+	repo := &fakeUploadRepo{}
+
+	id := "00000000000000000000000000000004"
+	now := time.Now()
+	sc := map[string]any{
+		"name":        "existing.txt",
+		"size":        5,
+		"sha256":      "sha",
+		"uploaded_at": now.Format(time.RFC3339Nano),
+	}
+	b, _ := json.Marshal(sc)
+	if err := os.WriteFile(filepath.Join(store.Root, id+".meta.json"), b, 0o600); err != nil {
+		t.Fatalf("create sidecar: %v", err)
+	}
+
+	// Pre-seed the repo so the sidecar is already indexed.
+	if err := repo.Create(context.Background(), id, "existing.txt", 5, "sha", now); err != nil {
+		t.Fatalf("seed repo: %v", err)
+	}
+
+	inserted, err := store.Backfill(context.Background(), repo)
+	if err != nil {
+		t.Fatalf("backfill: %v", err)
+	}
+	if inserted != 0 {
+		t.Errorf("inserted=%d, want 0", inserted)
+	}
+}