mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-17 04:36:50 +00:00
81ed379884
* volume server: route VolumeMarkReadonly to raft leader After a master raft election, volume servers may still heartbeat a follower while admin paths such as weed shell volume.mark call notifyMasterVolumeReadonly via vs.GetMaster(). Followers reject VolumeMarkReadonly with NotLeader, which breaks tiering and other mark-readonly workflows until the heartbeat loop reconnects. Resolve the leader through GetMasterConfiguration on configured -master peers (same Leader field filer/master clients already use) before calling VolumeMarkReadonly. When the leader differs from the heartbeat peer, update currentMaster so the heartbeat loop converges faster. Adds operation.LookupRaftLeaderMaster with unit tests. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: address review feedback on volume.mark raft leader routing Do not update currentMaster during leader lookup — heartbeat owns that field and uses stream GetLeader() to reconnect. Try the heartbeat peer first and only resolve the raft leader after a NotLeader rejection. Add ctx.Err() early exit and quieter logging for context cancellation. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(operation): thread the lookup timeout ctx into connection invalidation The 5s timeout drove only the RPC; WithMasterServerClient saw the unbounded outer ctx, so a self-inflicted timeout (slow GetMasterConfiguration during an election) was treated as a stale channel and tore down the shared master connection. Pass the timeout ctx into the helper so its own expiry leaves ctx.Err() set and spares the connection. --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>
63 lines
2.0 KiB
Go
63 lines
2.0 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
)
|
|
|
|
func TestLookupRaftLeaderMasterDoesNotUpdateCurrentMaster(t *testing.T) {
|
|
follower := startFakeMasterServerForLeaderLookup(t, &fakeLeaderConfigServer{
|
|
leader: "leader.example.com:9333.19333",
|
|
})
|
|
heartbeatPeer := pb.ServerAddress("10.0.0.1:9333")
|
|
vs := &VolumeServer{
|
|
SeedMasterNodes: []pb.ServerAddress{follower},
|
|
grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
store: &storage.Store{Ip: "127.0.0.1", Port: 8080},
|
|
}
|
|
vs.setCurrentMaster(heartbeatPeer)
|
|
|
|
leader, err := vs.lookupRaftLeaderMaster(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("lookup: %v", err)
|
|
}
|
|
if got, want := leader.ToHttpAddress(), "leader.example.com:9333"; got != want {
|
|
t.Fatalf("leader = %q, want %q", got, want)
|
|
}
|
|
if got := vs.getCurrentMaster(); got != heartbeatPeer {
|
|
t.Fatalf("currentMaster = %v, want unchanged %v", got, heartbeatPeer)
|
|
}
|
|
}
|
|
|
|
type fakeLeaderConfigServer struct {
|
|
master_pb.UnimplementedSeaweedServer
|
|
leader string
|
|
}
|
|
|
|
func (s *fakeLeaderConfigServer) GetMasterConfiguration(_ context.Context, _ *master_pb.GetMasterConfigurationRequest) (*master_pb.GetMasterConfigurationResponse, error) {
|
|
return &master_pb.GetMasterConfigurationResponse{Leader: s.leader}, nil
|
|
}
|
|
|
|
func startFakeMasterServerForLeaderLookup(t *testing.T, srv master_pb.SeaweedServer) pb.ServerAddress {
|
|
t.Helper()
|
|
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen: %v", err)
|
|
}
|
|
grpcServer := grpc.NewServer()
|
|
master_pb.RegisterSeaweedServer(grpcServer, srv)
|
|
go func() { _ = grpcServer.Serve(lis) }()
|
|
t.Cleanup(grpcServer.GracefulStop)
|
|
|
|
_, port, _ := net.SplitHostPort(lis.Addr().String())
|
|
return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
|
|
}
|