seaweedfs/weed/server/filer_grpc_server_chunk_condition_test.go

package weed_server

import (
	"context"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func chunkFid(fid string) *filer_pb.FileChunk {
	return &filer_pb.FileChunk{FileId: fid, Size: 100}
}

func ifChunksEqual(fids ...string) *filer_pb.WriteCondition {
	return one(&filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_CHUNKS_EQUAL, Fids: fids})
}

func TestChunkFidsEqualClause(t *testing.T) {
	withChunks := func(fids ...string) *filer.Entry {
		e := &filer.Entry{FullPath: "/test/obj"}
		for _, fid := range fids {
			e.Chunks = append(e.Chunks, chunkFid(fid))
		}
		return e
	}

	cases := []struct {
		name string
		cur  *filer.Entry
		fids []string
		want bool
	}{
		{"absent-no-chunks", nil, nil, true},
		{"absent-expected", nil, []string{"3,01"}, false},
		{"empty-expects-none", withChunks(), nil, true},
		{"exact", withChunks("3,01", "3,02"), []string{"3,01", "3,02"}, true},
		{"reordered", withChunks("3,01", "3,02"), []string{"3,02", "3,01"}, true},
		{"duplicates", withChunks("3,01", "3,01", "3,02"), []string{"3,02", "3,01", "3,01"}, true},
		// The strand: a concurrent update emptied the stored chunk list, so the
		// stale writer's expectation no longer holds.
		{"stored-emptied", withChunks(), []string{"3,01"}, false},
		// The reverse strand: the writer read no chunks, but a concurrent update
		// has since added one; an empty fids list still guards it.
		{"stored-filled", withChunks("3,01"), nil, false},
		{"chunk-added", withChunks("3,01", "3,02"), []string{"3,01"}, false},
		{"chunk-removed", withChunks("3,01"), []string{"3,01", "3,02"}, false},
		{"chunk-replaced", withChunks("3,09"), []string{"3,01"}, false},
		{"duplicate-count", withChunks("3,01"), []string{"3,01", "3,01"}, false},
	}
	for _, tc := range cases {
		if got := writeConditionSatisfied(ifChunksEqual(tc.fids...), tc.cur); got != tc.want {
			t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
		}
	}
}

// TestUpdateEntryChunkConditionPreventsStrand drives the UpdateEntry handler
// through the interleaving that strands an entry on a dead needle:
//
//  1. entry has chunks=[F]; needle F is live.
//  2. R (a chunk-preserving read-modify-write) snapshots chunks=[F].
//  3. D (eviction) commits chunks=[] first; F is diffed away and queued for
//     deletion. The store is seeded at chunks=[] to model D's commit.
//  4. R commits its stale snapshot last.
//
// Without a condition R's write wins and resurrects the reference to F; with
// IF_CHUNKS_EQUAL it fails with FailedPrecondition and the entry stays empty.
func TestUpdateEntryChunkConditionPreventsStrand(t *testing.T) {
	const fidF = "3,01637037d6"

	newServer := func(storedChunks ...*filer_pb.FileChunk) (*FilerServer, *renameTestStore) {
		store := newRenameTestStore()
		store.entries["/test/obj"] = &filer.Entry{
			FullPath: "/test/obj",
			Attr:     filer.Attr{Inode: 1, Mtime: time.Unix(1700000000, 0), Mode: 0644},
			Chunks:   storedChunks,
		}
		f := newRenameTestFiler(t, store)
		return &FilerServer{filer: f, option: &FilerOption{}, entryLockTable: util.NewLockTable[util.FullPath]()}, store
	}

	// R's stale request: it read chunks=[F] and writes back chunks=[F].
	staleReq := func(cond *filer_pb.WriteCondition) *filer_pb.UpdateEntryRequest {
		return &filer_pb.UpdateEntryRequest{
			Directory: "/test",
			Entry: &filer_pb.Entry{
				Name:       "obj",
				Attributes: &filer_pb.FuseAttributes{Mtime: 1700000005, FileMode: 0644, Inode: 1},
				Chunks:     []*filer_pb.FileChunk{chunkFid(fidF)},
			},
			Condition: cond,
		}
	}

	t.Run("unconditional-resurrects", func(t *testing.T) {
		fs, store := newServer()
		if _, err := fs.UpdateEntry(context.Background(), staleReq(nil)); err != nil {
			t.Fatalf("UpdateEntry: %v", err)
		}
		if got := len(store.entries["/test/obj"].GetChunks()); got != 1 {
			t.Fatalf("last-write-wins should have resurrected chunks=[F], got %d chunks", got)
		}
	})

	t.Run("condition-rejects-stale", func(t *testing.T) {
		fs, store := newServer()
		_, err := fs.UpdateEntry(context.Background(), staleReq(ifChunksEqual(fidF)))
		if status.Code(err) != codes.FailedPrecondition {
			t.Fatalf("stale write must fail with FailedPrecondition, got %v", err)
		}
		if got := len(store.entries["/test/obj"].GetChunks()); got != 0 {
			t.Fatalf("entry must stay chunks=[], got %d chunks", got)
		}
	})

	t.Run("condition-passes-fresh", func(t *testing.T) {
		fs, store := newServer(chunkFid(fidF))
		if _, err := fs.UpdateEntry(context.Background(), staleReq(ifChunksEqual(fidF))); err != nil {
			t.Fatalf("fresh read-modify-write must succeed: %v", err)
		}
		chunks := store.entries["/test/obj"].GetChunks()
		if len(chunks) != 1 || chunks[0].GetFileIdString() != fidF {
			t.Fatalf("chunks=[F] must be preserved, got %v", chunks)
		}
	})

	// The reverse strand: R read the entry before a concurrent writer added F,
	// so its stale write would wipe the chunk and queue F for deletion. An
	// IF_CHUNKS_EQUAL clause with no fids guards the emptiness it observed.
	t.Run("empty-expectation-rejects", func(t *testing.T) {
		fs, store := newServer(chunkFid(fidF))
		req := &filer_pb.UpdateEntryRequest{
			Directory: "/test",
			Entry: &filer_pb.Entry{
				Name:       "obj",
				Attributes: &filer_pb.FuseAttributes{Mtime: 1700000005, FileMode: 0644, Inode: 1},
			},
			Condition: ifChunksEqual(),
		}
		_, err := fs.UpdateEntry(context.Background(), req)
		if status.Code(err) != codes.FailedPrecondition {
			t.Fatalf("stale wipe must fail with FailedPrecondition, got %v", err)
		}
		if got := len(store.entries["/test/obj"].GetChunks()); got != 1 {
			t.Fatalf("chunks=[F] must be preserved, got %d chunks", got)
		}
	})
}

// DeleteEntry queues chunk deletions, so it must serialize on the same path
// lock the conditional writers hold; otherwise it can interleave with a
// passed precondition and the stale write resurrects the queued fids.
func TestDeleteEntryWaitsForPathLock(t *testing.T) {
	store := newRenameTestStore()
	store.entries["/test/obj"] = &filer.Entry{
		FullPath: "/test/obj",
		Attr:     filer.Attr{Inode: 1, Mtime: time.Unix(1700000000, 0), Mode: 0644},
	}
	f := newRenameTestFiler(t, store)
	fs := &FilerServer{filer: f, option: &FilerOption{}, entryLockTable: util.NewLockTable[util.FullPath]()}

	lockPath := util.FullPath("/test/obj")
	hold := fs.entryLockTable.AcquireLock("test", lockPath, util.ExclusiveLock)

	done := make(chan struct{})
	go func() {
		fs.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{Directory: "/test", Name: "obj"})
		close(done)
	}()

	select {
	case <-done:
		t.Fatal("DeleteEntry completed while the path lock was held")
	case <-time.After(100 * time.Millisecond):
	}

	fs.entryLockTable.ReleaseLock(lockPath, hold)
	<-done
	if _, ok := store.entries["/test/obj"]; ok {
		t.Fatal("entry not deleted after the lock was released")
	}
}