seaweedfs/weed/server/filer_grpc_server_create_route_test.go

package weed_server

import (
	"context"
	"fmt"
	"net"
	"testing"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/peer"
	"google.golang.org/grpc/status"

	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// createRouteServer builds a filer whose ring holds two unreachable peers, so a
// forwarded create fails at the dial and is distinguishable from a local apply.
func createRouteServer(t *testing.T) (*FilerServer, *renameTestStore) {
	t.Helper()
	const self = pb.ServerAddress("127.0.0.1:18888")
	store := newRenameTestStore()
	f := newRenameTestFiler(t, store)
	f.DirBucketsPath = "/buckets"
	f.Dlm = lock_manager.NewDistributedLockManager(self)
	f.Dlm.LockRing.SetSnapshot([]pb.ServerAddress{self, "127.0.0.1:18889"}, 1)
	return &FilerServer{
		filer:          f,
		option:         &FilerOption{Host: self},
		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
		entryLockTable: util.NewLockTable[util.FullPath](),
	}, store
}

// peerOwnedName returns a child name under /test whose entry the ring assigns to
// the peer, so a routed create must leave this filer.
func peerOwnedName(t *testing.T, fs *FilerServer) string {
	t.Helper()
	for i := 0; i < 4000; i++ {
		name := fmt.Sprintf("obj-%d", i)
		key := entryRouteKey(util.NewFullPath("/test", name))
		if fs.filer.Dlm.LockRing.GetPrimary(key) != fs.option.Host {
			return name
		}
	}
	t.Skip("no name owned by the peer")
	return ""
}

func createReq(name string, oExcl bool) *filer_pb.CreateEntryRequest {
	return &filer_pb.CreateEntryRequest{
		Directory:                "/test",
		OExcl:                    oExcl,
		SkipCheckParentDirectory: true,
		Entry: &filer_pb.Entry{
			Name:       name,
			Attributes: &filer_pb.FuseAttributes{Mtime: 1700000000, FileMode: 0644, Inode: 1},
		},
	}
}

// An exclusive create for a peer-owned path must leave this filer rather than
// take the local lock, whose arbitration would not bind the other filers.
func TestCreateEntryExclusiveLeavesNonOwner(t *testing.T) {
	fs, store := createRouteServer(t)
	name := peerOwnedName(t, fs)

	_, err := fs.CreateEntry(context.Background(), createReq(name, true))
	if err == nil {
		t.Fatal("exclusive create on a non-owner must not be applied locally")
	}
	if _, found := store.entries[string(util.NewFullPath("/test", name))]; found {
		t.Fatal("forwarded create must not also write locally")
	}
}

// Plain creates are upserts whichever filer applies them, so routing them would
// buy nothing and cost a hop.
func TestCreateEntryPlainStaysLocal(t *testing.T) {
	fs, store := createRouteServer(t)
	name := peerOwnedName(t, fs)

	resp, err := fs.CreateEntry(context.Background(), createReq(name, false))
	if err != nil || resp.Error != "" {
		t.Fatalf("plain create must be applied locally, got err=%v resp=%v", err, resp.Error)
	}
	if _, found := store.entries[string(util.NewFullPath("/test", name))]; !found {
		t.Fatal("plain create must reach the local store")
	}
}

// ringPeerCtx simulates a request arriving over a connection from a ring
// member, the provenance an is_moved marker needs before it is trusted.
func ringPeerCtx() context.Context {
	return peer.NewContext(context.Background(), &peer.Peer{
		Addr: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 49999},
	})
}

// is_moved bounds forwarding to one hop: the owner applies it even if its own
// ring view says someone else owns the key.
func TestCreateEntryMovedAppliesLocally(t *testing.T) {
	fs, store := createRouteServer(t)
	name := peerOwnedName(t, fs)

	req := createReq(name, true)
	req.IsMoved = true
	resp, err := fs.CreateEntry(ringPeerCtx(), req)
	if err != nil || resp.Error != "" {
		t.Fatalf("forwarded create must be applied locally, got err=%v resp=%v", err, resp.Error)
	}
	if _, found := store.entries[string(util.NewFullPath("/test", name))]; !found {
		t.Fatal("forwarded create must reach the local store")
	}
}

// A caller can set is_moved, but without a ring member's connection it is not
// trusted: on a non-owner the request is refused rather than evaluated under
// the wrong lock or forwarded again, where it could cycle while rings disagree.
func TestCreateEntryMovedFromClientIsNotTrusted(t *testing.T) {
	fs, store := createRouteServer(t)
	name := peerOwnedName(t, fs)

	req := createReq(name, true)
	req.IsMoved = true
	_, err := fs.CreateEntry(context.Background(), req)
	if status.Code(err) != codes.PermissionDenied {
		t.Fatalf("a forged is_moved must be refused, got err=%v", err)
	}
	if _, found := store.entries[string(util.NewFullPath("/test", name))]; found {
		t.Fatal("a forged is_moved must not cause a local write")
	}
}

// Forwarding an unlock to an unreachable owner must surface the RPC error, not
// panic dereferencing the nil response of the failed call.
func TestDistributedUnlockFailedForwardReturnsError(t *testing.T) {
	fs, _ := createRouteServer(t)

	var name string
	for i := 0; i < 4000; i++ {
		candidate := fmt.Sprintf("lock-%d", i)
		if fs.filer.Dlm.LockRing.GetPrimary(candidate) != fs.option.Host {
			name = candidate
			break
		}
	}
	if name == "" {
		t.Skip("no lock owned by the peer")
	}

	resp, err := fs.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
		Name:       name,
		RenewToken: "tok",
	})
	if err != nil {
		t.Fatalf("unlock forward should report via resp.Error, not err: %v", err)
	}
	if resp.Error == "" {
		t.Fatal("a failed forward must surface an error")
	}
}

// An exclusive create this filer owns is applied locally, under its per-path lock.
func TestCreateEntryExclusiveAppliedByOwner(t *testing.T) {
	fs, store := createRouteServer(t)
	var name string
	for i := 0; i < 4000 && name == ""; i++ {
		candidate := fmt.Sprintf("obj-%d", i)
		key := entryRouteKey(util.NewFullPath("/test", candidate))
		if fs.filer.Dlm.LockRing.GetPrimary(key) == fs.option.Host {
			name = candidate
		}
	}
	if name == "" {
		t.Skip("no name owned by this filer")
	}

	resp, err := fs.CreateEntry(context.Background(), createReq(name, true))
	if err != nil || resp.Error != "" {
		t.Fatalf("owner must apply the create, got err=%v resp=%v", err, resp.Error)
	}
	if _, found := store.entries[string(util.NewFullPath("/test", name))]; !found {
		t.Fatal("owner's create must reach the local store")
	}
}

// A key still inside the cooling-off window belongs to its prior owner. If that
// filer cannot be reached the create fails: it may be partitioned rather than
// down, and applying here would be the second serialization point on the key.
func TestCreateEntryFailsWhenPriorOwnerIsUnreachable(t *testing.T) {
	const self = pb.ServerAddress("127.0.0.1:28888")
	const peer = pb.ServerAddress("127.0.0.1:28889")
	store := newRenameTestStore()
	f := newRenameTestFiler(t, store)
	f.DirBucketsPath = "/buckets"
	f.Dlm = lock_manager.NewDistributedLockManager(self)
	// Dropping the peer moves every key it owned to this filer, with the peer as
	// their prior owner — and nothing is listening on it.
	f.Dlm.LockRing.SetSnapshot([]pb.ServerAddress{self, peer}, 1)
	f.Dlm.LockRing.SetSnapshot([]pb.ServerAddress{self}, 2)
	fs := &FilerServer{
		filer:          f,
		option:         &FilerOption{Host: self},
		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
		entryLockTable: util.NewLockTable[util.FullPath](),
	}

	var name string
	for i := 0; i < 4000 && name == ""; i++ {
		candidate := fmt.Sprintf("obj-%d", i)
		if fs.filer.Dlm.LockRing.PriorOwner(entryRouteKey(util.NewFullPath("/test", candidate))) == peer {
			name = candidate
		}
	}
	if name == "" {
		t.Skip("no key whose prior owner is the departed peer")
	}

	resp, err := fs.CreateEntry(context.Background(), createReq(name, true))
	if err == nil {
		t.Fatalf("unreachable prior owner must fail the create, got resp=%v", resp)
	}
	if resp == nil {
		t.Fatal("a failed create must still answer with a response, not nil")
	}
	if _, found := store.entries[string(util.NewFullPath("/test", name))]; found {
		t.Fatal("a create routed away must not also write locally")
	}
}