seaweedfs/weed/server/filer_grpc_server_route.go

package weed_server

import (
	"context"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/peer"
	"google.golang.org/grpc/status"
)

// writeOwner returns the filer that serializes writes to key, or "" when this
// filer is the serialization point — because it owns the key, or because there
// is no ring and every filer applies locally.
//
// A ring change hands a key to its new owner before that owner has rebuilt the
// locks the prior owner still holds, so the prior owner keeps the key until the
// cooling-off window closes.
func (fs *FilerServer) writeOwner(key string) pb.ServerAddress {
	if fs.filer.Dlm == nil {
		return ""
	}
	owner := fs.filer.Dlm.LockRing.WriteOwner(key)
	if owner == fs.option.Host {
		return ""
	}
	return owner
}

// forwardToWriteOwner sends the request to key's write owner so a single filer's
// per-path lock arbitrates every writer of that key. handled=false means this
// filer is the owner and the caller should apply the request locally.
//
// An unreachable owner fails the request; it is never re-sent to another filer.
// gRPC reports a response lost in transit as Unavailable, indistinguishable from
// one the owner never saw, so a retry elsewhere could re-apply what the owner
// already committed — and an owner unreachable from here may be partitioned
// rather than down, still serving the key to everyone else. The ring hands the
// key on when the cooling-off window closes, so the outage is bounded.
func (fs *FilerServer) forwardToWriteOwner(ctx context.Context, key string, send func(owner pb.ServerAddress) error) (handled bool, err error) {
	owner := fs.writeOwner(key)
	if owner == "" {
		return false, nil
	}
	if err := send(owner); err != nil {
		glog.V(1).InfofCtx(ctx, "route %s to owner %s: %v", key, owner, err)
		return true, err
	}
	return true, nil
}

// entryRouteKey is the ring key for an entry's writes. It shares the S3
// gateway's namespace so an object's ObjectTransaction and its CreateEntry
// resolve to the same owner, and land on that filer's one per-path lock.
func entryRouteKey(fullpath util.FullPath) string {
	return s3_constants.ObjectWriteRouteKeyPrefix + string(fullpath)
}

// movedFromPeer reports whether an is_moved marker arrived on a connection
// from a ring member, i.e. it marks a genuine forwarded hop.
func (fs *FilerServer) movedFromPeer(ctx context.Context, isMoved bool) bool {
	if !isMoved || fs.filer.Dlm == nil {
		return false
	}
	p, ok := peer.FromContext(ctx)
	if !ok {
		return false
	}
	peerHost, _, err := net.SplitHostPort(p.Addr.String())
	if err != nil {
		return false
	}
	peerIP := net.ParseIP(peerHost)
	if peerIP == nil {
		return false
	}
	for _, ip := range fs.ringMemberIPs(ctx) {
		if ip.Equal(peerIP) {
			return true
		}
	}
	return false
}

// ringPeerIPs caches resolved member addresses of one ring membership. A
// member's hostname may re-resolve under the same ring address, so the cache
// expires rather than trusting the resolution forever.
type ringPeerIPs struct {
	members string
	ips     []net.IP
	expires time.Time
}

const ringPeerIPTTL = 5 * time.Minute

// ringMemberIPs returns the ring members' addresses as IPs. Members can
// advertise hostnames, so resolution is cached per membership to keep DNS off
// each forwarded request; a failed lookup is not cached, so a DNS blip does
// not keep rejecting genuine forwards until the next ring change.
func (fs *FilerServer) ringMemberIPs(ctx context.Context) []net.IP {
	members := fs.filer.Dlm.LockRing.GetSnapshot()
	var sb strings.Builder
	for _, member := range members {
		sb.WriteString(string(member))
		sb.WriteByte(' ')
	}
	key := sb.String()
	if cached := fs.ringPeerIPs.Load(); cached != nil && cached.members == key && time.Now().Before(cached.expires) {
		return cached.ips
	}
	resolved, _, _ := fs.ringResolveGroup.Do(key, func() (any, error) {
		if cached := fs.ringPeerIPs.Load(); cached != nil && cached.members == key && time.Now().Before(cached.expires) {
			return cached.ips, nil
		}
		// Shared by every caller waiting on this key: the lookups outlive the
		// first request's cancellation, and run in parallel so one slow member
		// cannot starve the rest of the shared deadline.
		var ips []net.IP
		var hosts []string
		for _, member := range members {
			host, _, err := net.SplitHostPort(string(member))
			if err != nil {
				continue
			}
			if ip := net.ParseIP(host); ip != nil {
				ips = append(ips, ip)
				continue
			}
			hosts = append(hosts, host)
		}
		var wg sync.WaitGroup
		var mu sync.Mutex
		failed := false
		for _, host := range hosts {
			wg.Add(1)
			go func(host string) {
				defer wg.Done()
				lookupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
				defer cancel()
				found, err := net.DefaultResolver.LookupIP(lookupCtx, "ip", host)
				mu.Lock()
				defer mu.Unlock()
				if err != nil {
					failed = true
					return
				}
				ips = append(ips, found...)
			}(host)
		}
		wg.Wait()
		if !failed {
			fs.ringPeerIPs.Store(&ringPeerIPs{members: key, ips: ips, expires: time.Now().Add(ringPeerIPTTL)})
		}
		return ips, nil
	})
	return resolved.([]net.IP)
}

// checkMovedMarker refuses a request whose is_moved marker did not arrive from
// a ring member while this filer is not the key's owner. The marker is
// caller-controlled: applying it would evaluate a conditional mutation under a
// non-owner's lock, and re-forwarding a claimed hop can cycle while rings
// disagree — so an unverifiable marker on a non-owner is refused instead.
// PermissionDenied keeps the refusal distinct from a write condition's
// FailedPrecondition, which callers use to detect a stale stamp.
// owner=="" means this filer is the serialization point and the request can
// be applied locally.
func (fs *FilerServer) checkMovedMarker(ctx context.Context, isMoved bool, owner pb.ServerAddress) error {
	if !isMoved || owner == "" || owner == fs.option.Host || fs.movedFromPeer(ctx, isMoved) {
		return nil
	}
	return status.Errorf(codes.PermissionDenied, "is_moved not sent by a ring member; the key's owner is %s", owner)
}