seaweedfs/weed/pb/grpc_client_cascade_test.go

package pb

import (
	"context"
	"errors"
	"io"
	"net"
	"net/http"
	"sync"
	"testing"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)

// cascadeFilerServer answers a "slow" lookup only once its caller is known to be
// in flight, so a second caller can fail while the first still holds the shared
// cached ClientConn.
type cascadeFilerServer struct {
	filer_pb.UnimplementedSeaweedFilerServer
	inFlight  chan struct{}
	release   chan struct{}
	closeOnce sync.Once
}

func (s *cascadeFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
	if req.Name == "slow" {
		s.closeOnce.Do(func() { close(s.inFlight) })
		select {
		case <-s.release:
		case <-ctx.Done():
			return nil, ctx.Err()
		}
	}
	return &filer_pb.LookupDirectoryEntryResponse{}, nil
}

// startCascadeFiler serves a fake filer and resets the process-wide connection
// cache so the test owns the shared ClientConn under exercise.
func startCascadeFiler(t *testing.T) (*cascadeFilerServer, string, grpc.DialOption) {
	t.Helper()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	fake := &cascadeFilerServer{inFlight: make(chan struct{}), release: make(chan struct{})}
	server := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(server, fake)
	go server.Serve(listener)

	grpcClientsLock.Lock()
	previous := grpcClients
	grpcClients = make(map[string]*versionedGrpcClient)
	grpcClientsLock.Unlock()
	t.Cleanup(func() {
		grpcClientsLock.Lock()
		cached := grpcClients
		grpcClients = previous
		grpcClientsLock.Unlock()
		for _, connection := range cached {
			connection.Close()
		}
		server.Stop()
	})

	return fake, listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())
}

// TestWithGrpcClient_AbandonedRequestKeepsSharedConnection reproduces
// seaweedfs#10947 end to end: an S3-style caller that hands WithGrpcClient a
// context.Background() while its RPC rides the (now abandoned) HTTP request
// context used to close the shared filer ClientConn, killing every concurrent
// upload on it with "the client connection is closing".
func TestWithGrpcClient_AbandonedRequestKeepsSharedConnection(t *testing.T) {
	fake, address, dialOption := startCascadeFiler(t)

	var concurrentErr error
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		concurrentErr = WithGrpcClient(context.Background(), false, 0, func(connection *grpc.ClientConn) error {
			_, err := filer_pb.NewSeaweedFilerClient(connection).LookupDirectoryEntry(context.Background(),
				&filer_pb.LookupDirectoryEntryRequest{Directory: "/buckets/b", Name: "slow"})
			return err
		}, address, false, dialOption)
	}()
	<-fake.inFlight

	abandoned, cancel := context.WithCancel(context.Background())
	cancel()
	if err := WithGrpcClient(context.Background(), false, 0, func(connection *grpc.ClientConn) error {
		_, err := filer_pb.NewSeaweedFilerClient(connection).LookupDirectoryEntry(abandoned,
			&filer_pb.LookupDirectoryEntryRequest{Directory: "/buckets/b", Name: "fast"})
		return err
	}, address, false, dialOption); err == nil {
		t.Fatal("the abandoned request should have failed")
	}

	close(fake.release)
	wg.Wait()
	if concurrentErr != nil {
		t.Fatalf("concurrent caller must survive an unrelated abandoned request: %v", concurrentErr)
	}
}

func (s *cascadeFilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeMetadataServer) error {
	if !req.ClientSupportsMetadataChunks {
		return nil
	}
	return stream.Send(&filer_pb.SubscribeMetadataResponse{
		LogFileRefs: []*filer_pb.LogFileChunkRef{{
			FilerId:  "5319c0e8",
			FileTsNs: time.Now().UnixNano(),
			Chunks:   []*filer_pb.FileChunk{{FileId: "4471,ce598bfff8416e", Size: 64}},
		}},
	})
}

// TestWithGrpcClient_EndedStreamKeepsSharedConnection covers the other half of
// seaweedfs#10947: the S3 gateway follows filer metadata on a long-lived
// SubscribeMetadata stream and reconnects forever. Every ordinary end of that
// stream used to drop the shared filer ClientConn the request handlers use,
// firing the same burst of "the client connection is closing" failures.
func TestWithGrpcClient_EndedStreamKeepsSharedConnection(t *testing.T) {
	fake, address, dialOption := startCascadeFiler(t)

	var concurrentErr error
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		concurrentErr = WithGrpcClient(context.Background(), false, 0, func(connection *grpc.ClientConn) error {
			_, err := filer_pb.NewSeaweedFilerClient(connection).LookupDirectoryEntry(context.Background(),
				&filer_pb.LookupDirectoryEntryRequest{Directory: "/buckets/b", Name: "slow"})
			return err
		}, address, false, dialOption)
	}()
	<-fake.inFlight

	if err := WithGrpcClient(context.Background(), true, 0, func(connection *grpc.ClientConn) error {
		stream, err := filer_pb.NewSeaweedFilerClient(connection).SubscribeMetadata(context.Background(), &filer_pb.SubscribeMetadataRequest{})
		if err != nil {
			return err
		}
		_, err = stream.Recv()
		return err
	}, address, false, dialOption); err == nil {
		t.Fatal("the ended stream should have reported io.EOF")
	}

	close(fake.release)
	wg.Wait()
	if concurrentErr != nil {
		t.Fatalf("concurrent caller must survive an unrelated stream ending: %v", concurrentErr)
	}
}

// TestWithGrpcClient_LogChunkReadFailureKeepsSharedConnection covers a
// subscriber replaying persisted log chunks over HTTP: an unreachable volume
// server reports "connection refused" as the subscription's error, which used
// to drop the shared filer ClientConn and cancel the assign and upload RPCs
// riding on it.
func TestWithGrpcClient_LogChunkReadFailureKeepsSharedConnection(t *testing.T) {
	fake, address, dialOption := startCascadeFiler(t)

	var concurrentErr error
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		concurrentErr = WithGrpcClient(context.Background(), false, 0, func(connection *grpc.ClientConn) error {
			_, err := filer_pb.NewSeaweedFilerClient(connection).LookupDirectoryEntry(context.Background(),
				&filer_pb.LookupDirectoryEntryRequest{Directory: "/buckets/b", Name: "slow"})
			return err
		}, address, false, dialOption)
	}()
	<-fake.inFlight

	option := &MetadataFollowOption{
		ClientName:     "mount",
		PathPrefix:     "/",
		EventErrorType: DontLogError,
		LogFileReaderFn: func(chunks []*filer_pb.FileChunk) (io.ReadCloser, error) {
			// An unreachable volume server, as a filerProxy mount sees it.
			response, err := http.Get("http://127.0.0.1:1/" + chunks[0].FileId)
			if err != nil {
				return nil, err
			}
			return response.Body, nil
		},
	}
	subscribe := makeSubscribeMetadataFunc(option, func(resp *filer_pb.SubscribeMetadataResponse) error { return nil })
	err := WithGrpcClient(context.Background(), true, 0, func(connection *grpc.ClientConn) error {
		return subscribe(filer_pb.NewSeaweedFilerClient(connection))
	}, address, false, dialOption)
	if err == nil || !errors.Is(err, ErrLogFileRead) {
		t.Fatalf("the replay should have failed reading the log chunk: %v", err)
	}

	close(fake.release)
	wg.Wait()
	if concurrentErr != nil {
		t.Fatalf("concurrent caller must survive a log chunk read failure: %v", concurrentErr)
	}
}