seaweedfs/weed/topology/store_replicate_test.go

package topology

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/operation"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

// TestDistributedOperationCancelsSiblingsOnFirstError verifies that once one
// replica fails, an outstanding replica still stalled in a dial timeout is
// cancelled rather than gating the caller until it times out.
func TestDistributedOperationCancelsSiblingsOnFirstError(t *testing.T) {
	locations := []operation.Location{{Url: "fast"}, {Url: "slow"}}
	cancelled := make(chan struct{}, 1)

	start := time.Now()
	err := DistributedOperation(context.Background(), locations, func(ctx context.Context, location operation.Location) error {
		if location.Url == "fast" {
			return errors.New("connection refused")
		}
		// slow: a replica stalled in a dial timeout
		select {
		case <-ctx.Done():
			cancelled <- struct{}{}
			return ctx.Err()
		case <-time.After(10 * time.Second):
			return nil
		}
	})

	if err == nil {
		t.Fatal("expected an error from the fast-failing replica")
	}
	if elapsed := time.Since(start); elapsed > 2*time.Second {
		t.Fatalf("did not fail fast: took %v", elapsed)
	}
	select {
	case <-cancelled:
	case <-time.After(2 * time.Second):
		t.Fatal("slow replica was not cancelled after the first error")
	}
}

func TestDistributedOperationEmpty(t *testing.T) {
	err := DistributedOperation(context.Background(), nil, func(ctx context.Context, location operation.Location) error {
		t.Fatal("op should not be called when there are no locations")
		return nil
	})
	if err != nil {
		t.Fatalf("expected nil for no locations, got %v", err)
	}
}

type mockMasterServer struct {
	master_pb.UnimplementedSeaweedServer
	mu        sync.Mutex
	calls     int
	locations []*master_pb.Location
}

func (m *mockMasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.calls++
	var vls []*master_pb.LookupVolumeResponse_VolumeIdLocation
	for _, vid := range req.VolumeOrFileIds {
		vl := &master_pb.LookupVolumeResponse_VolumeIdLocation{
			VolumeOrFileId: vid,
			Locations:      m.locations,
		}
		if len(m.locations) == 0 {
			vl.Error = fmt.Sprintf("volume id %s not found", vid)
		}
		vls = append(vls, vl)
	}
	return &master_pb.LookupVolumeResponse{VolumeIdLocations: vls}, nil
}

func startMockMasterServer(t *testing.T, master *mockMasterServer) (operation.GetMasterFn, grpc.DialOption) {
	t.Helper()
	lis, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	grpcServer := grpc.NewServer()
	master_pb.RegisterSeaweedServer(grpcServer, master)
	serveErr := make(chan error, 1)
	go func() { serveErr <- grpcServer.Serve(lis) }()
	t.Cleanup(func() {
		grpcServer.Stop()
		if err := <-serveErr; err != nil && !errors.Is(err, grpc.ErrServerStopped) {
			t.Errorf("mock master serve: %v", err)
		}
	})

	grpcPort := lis.Addr().(*net.TCPAddr).Port
	return func(_ context.Context) pb.ServerAddress {
		return pb.NewServerAddressWithGrpcPort(fmt.Sprintf("127.0.0.1:%d", grpcPort), grpcPort)
	}, grpc.WithTransportCredentials(insecure.NewCredentials())
}

func TestGetWritableRemoteReplicationsRefreshesReadOnlyReplicas(t *testing.T) {
	master := &mockMasterServer{locations: []*master_pb.Location{
		{Url: "127.0.0.1:8080"},
		{Url: "127.0.0.2:8080"},
	}}
	masterFn, dialOption := startMockMasterServer(t, master)
	store := &storage.Store{Ip: "127.0.0.1", Port: 8080}
	volumeId := needle.VolumeId(1)
	operation.InvalidateVolumeIdLocationCache(volumeId.String())

	locations, err := GetWritableRemoteReplications(store, dialOption, volumeId, masterFn)
	if err != nil {
		t.Fatalf("first lookup: %v", err)
	}
	if len(locations) != 1 || locations[0].Url != "127.0.0.2:8080" {
		t.Fatalf("first lookup locations = %v", locations)
	}

	master.mu.Lock()
	master.locations = []*master_pb.Location{
		{Url: "127.0.0.1:8080"},
		{Url: "127.0.0.2:8080", ReadOnly: true, ReadOnlyCanDelete: true},
	}
	master.mu.Unlock()

	locations, err = GetWritableRemoteReplications(store, dialOption, volumeId, masterFn)
	if err != nil {
		t.Fatalf("second lookup: %v", err)
	}
	if len(locations) != 0 {
		t.Fatalf("read-only replica remained a write target: %v", locations)
	}
	master.mu.Lock()
	calls := master.calls
	master.mu.Unlock()
	if calls != 2 {
		t.Fatalf("master lookup calls = %d, want 2", calls)
	}

	locations, err = GetRemoteReplications(store, dialOption, volumeId, masterFn)
	if err != nil {
		t.Fatalf("delete lookup: %v", err)
	}
	if len(locations) != 1 || !locations[0].ReadOnly || !locations[0].ReadOnlyCanDelete {
		t.Fatalf("read-only-can-delete target was dropped: %v", locations)
	}
}

// TestReplicatedWriteForwardsFsyncToReplicas verifies that the fsync=true
// request parameter is forwarded to replica volume servers in the fan-out
// request, so a durable write means every replica has flushed to disk.
func TestReplicatedWriteForwardsFsyncToReplicas(t *testing.T) {
	replicaQueries := make(chan url.Values, 4)
	replica := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		replicaQueries <- r.URL.Query()
		w.WriteHeader(http.StatusCreated)
		if _, err := w.Write([]byte(`{"size":1}`)); err != nil {
			t.Errorf("replica write response: %v", err)
		}
	}))
	defer replica.Close()
	replicaHost := strings.TrimPrefix(replica.URL, "http://")

	masterFn, dialOption := startMockMasterServer(t, &mockMasterServer{
		locations: []*master_pb.Location{{Url: replicaHost}},
	})

	store := &storage.Store{}
	volumeId := needle.VolumeId(1)

	for _, tc := range []struct {
		name      string
		fsync     string
		wantFsync bool
	}{
		{name: "fsync requested", fsync: "true", wantFsync: true},
		{name: "no fsync requested", fsync: "", wantFsync: false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			operation.InvalidateVolumeIdLocationCache(volumeId.String())

			path := "http://127.0.0.1:8080/1,01637037d6"
			if tc.fsync != "" {
				path += "?fsync=" + tc.fsync
			}
			r := httptest.NewRequest(http.MethodPost, path, nil)
			if err := r.ParseForm(); err != nil {
				t.Fatal(err)
			}

			n := &needle.Needle{
				Id:   1,
				Data: []byte("test data"),
				Ttl:  needle.EMPTY_TTL,
			}
			if _, err := ReplicatedWrite(context.Background(), masterFn, dialOption, store, volumeId, n, r, ""); err != nil {
				t.Fatalf("ReplicatedWrite: %v", err)
			}

			select {
			case q := <-replicaQueries:
				got := q.Get("fsync")
				if tc.wantFsync && got != "true" {
					t.Errorf("expected fsync=true in replica query, got %q (query: %v)", got, q)
				}
				if !tc.wantFsync && got != "" {
					t.Errorf("expected no fsync in replica query, got %q (query: %v)", got, q)
				}
			case <-time.After(5 * time.Second):
				t.Fatal("replica never received the fan-out request")
			}
		})
	}
}

// TestReplicatedWriteRejectsWriteWithNoTarget verifies that a write this server
// cannot store and has no replica to forward to fails instead of being
// acknowledged with nothing written.
func TestReplicatedWriteRejectsWriteWithNoTarget(t *testing.T) {
	master := &mockMasterServer{}
	masterFn, dialOption := startMockMasterServer(t, master)
	store := &storage.Store{Ip: "127.0.0.1", Port: 8080}
	volumeId := needle.VolumeId(31234)

	for _, tc := range []struct {
		name      string
		query     string
		locations []*master_pb.Location
	}{
		{name: "volume unknown to the master"},
		{name: "master lists only this server", locations: []*master_pb.Location{{Url: "127.0.0.1:8080"}}},
		{name: "replica write for a volume not on this server", query: "?type=replicate"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			master.mu.Lock()
			master.locations = tc.locations
			master.mu.Unlock()

			r := httptest.NewRequest(http.MethodPost, "http://127.0.0.1:8080/31234,01637037d6"+tc.query, nil)
			if err := r.ParseForm(); err != nil {
				t.Fatal(err)
			}
			n := &needle.Needle{
				Id:   1,
				Data: []byte("test data"),
				Ttl:  needle.EMPTY_TTL,
			}
			_, err := ReplicatedWrite(context.Background(), masterFn, dialOption, store, volumeId, n, r, "")
			if err == nil {
				t.Fatal("ReplicatedWrite acknowledged a write that no volume stored")
			}
			if !strings.Contains(err.Error(), "not found") {
				t.Fatalf("unexpected error: %v", err)
			}
		})
	}
}