mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-30 04:37:07 +00:00
* wdclient: bound the wait for a master leader by the caller's context WithClient waited on GetMaster with context.Background(), so a caller that arrived while no master leader was known parked in a 200ms poll loop until one appeared, whatever deadline it had already set on the RPC. Each retry above it then left another goroutine in the same wait. Take the context in WithClient and WithClientCustomGetMaster and hand it to GetMaster, and stop the retry loop once it is done. The dial keeps context.Background(): fn brings its own RPC context, so a cancellation seen here cannot be attributed to the shared connection. Call sites pass whatever they hold: the request context in the filer's CollectionList, DeleteCollection and Statistics handlers and in the credential store's propagation, the operation context in the shell's s3.bucket.delete and the kafka gateway's broker and filer discovery, and context.Background() where there is none - the shell commands, the admin dashboard wrapper, and the exclusive locker's initial lease. The locker's release keeps its own uncancelled context so a slow unlock cannot turn into a ghost lock. Claude-Session: https://claude.ai/code/session_01BjDWtZsCoZY6x4pdDmGWxU * wdclient: test that WithClient gives up with the caller's context Claude-Session: https://claude.ai/code/session_01BjDWtZsCoZY6x4pdDmGWxU * wdclient: cut the master retry backoff short when the caller gives up util.Retry sleeps unconditionally between attempts, so a transient error arriving just before the caller's deadline still cost it a full backoff step. Use the context-aware util.RetryWithBackoff, the same helper the volume lookup in this file already uses. Two call sites went with it: the shell's lock-holder lookup builds its three second bound before WithClient so it also covers finding the leader, as its comment already promised, and the filer's post-delete collection cleanup goes back to an uncancelled context - the entry is already gone, so a caller that hung up must not leave the collection behind. Claude-Session: https://claude.ai/code/session_01BjDWtZsCoZY6x4pdDmGWxU * wdclient: test that a cancel during backoff ends the retry Claude-Session: https://claude.ai/code/session_01BjDWtZsCoZY6x4pdDmGWxU
144 lines
4.6 KiB
Go
144 lines
4.6 KiB
Go
package shell
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
)
|
|
|
|
func init() {
|
|
Commands = append(Commands, &commandRaftLeaderTransfer{})
|
|
}
|
|
|
|
type commandRaftLeaderTransfer struct{}
|
|
|
|
func (c *commandRaftLeaderTransfer) Name() string {
|
|
return "cluster.raft.transferLeader"
|
|
}
|
|
|
|
func (c *commandRaftLeaderTransfer) Help() string {
|
|
return `transfer raft leadership to another master server
|
|
|
|
This command initiates a graceful leadership transfer from the current
|
|
leader to another server. Use this before performing maintenance on
|
|
the current leader to reduce errors in filers and other components.
|
|
|
|
Examples:
|
|
# Transfer to any eligible follower (auto-selection)
|
|
cluster.raft.transferLeader
|
|
|
|
# Transfer to a specific server
|
|
cluster.raft.transferLeader -id <server_id> -address <server_grpc_address>
|
|
|
|
Notes:
|
|
- Requires hashicorp raft (-raftHashicorp=true on master)
|
|
- This command must be sent to the current leader
|
|
- The target server must be a voting member of the raft cluster
|
|
- Use 'cluster.raft.ps' to list available servers and identify the leader
|
|
`
|
|
}
|
|
|
|
func (c *commandRaftLeaderTransfer) HasTag(CommandTag) bool {
|
|
return false
|
|
}
|
|
|
|
func (c *commandRaftLeaderTransfer) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
|
leaderTransferCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
|
targetId := leaderTransferCommand.String("id", "", "target server id (must be used with -address)")
|
|
targetAddress := leaderTransferCommand.String("address", "", "target server grpc address (must be used with -id)")
|
|
|
|
if err := leaderTransferCommand.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Validate: id and address must be specified together
|
|
if *targetId != "" && *targetAddress == "" {
|
|
return fmt.Errorf("-address is required when -id is specified")
|
|
}
|
|
if *targetAddress != "" && *targetId == "" {
|
|
return fmt.Errorf("-id is required when -address is specified")
|
|
}
|
|
|
|
// First, show current cluster status
|
|
fmt.Fprintf(writer, "Checking current raft cluster status...\n")
|
|
|
|
var currentLeader string
|
|
err := commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
resp, err := client.RaftListClusterServers(ctx, &master_pb.RaftListClusterServersRequest{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to list cluster servers: %v", err)
|
|
}
|
|
|
|
if len(resp.ClusterServers) == 0 {
|
|
fmt.Fprintf(writer, "No raft cluster configured (single master mode)\n")
|
|
return fmt.Errorf("leadership transfer not available in single master mode")
|
|
}
|
|
|
|
fmt.Fprintf(writer, "Raft cluster has %d servers:\n", len(resp.ClusterServers))
|
|
for _, server := range resp.ClusterServers {
|
|
suffix := ""
|
|
if server.IsLeader {
|
|
suffix = " <- current leader"
|
|
currentLeader = server.Id
|
|
}
|
|
fmt.Fprintf(writer, " %s %s [%s]%s\n", server.Id, server.Address, server.Suffrage, suffix)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if currentLeader == "" {
|
|
return fmt.Errorf("no leader found in cluster")
|
|
}
|
|
|
|
// Perform the transfer
|
|
targetDesc := "any eligible follower"
|
|
if *targetId != "" {
|
|
targetDesc = fmt.Sprintf("server %s (%s)", *targetId, *targetAddress)
|
|
}
|
|
fmt.Fprintf(writer, "\nTransferring leadership from %s to %s...\n", currentLeader, targetDesc)
|
|
|
|
err = commandEnv.MasterClient.WithClient(context.Background(), true, func(client master_pb.SeaweedClient) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
resp, err := client.RaftLeadershipTransfer(ctx, &master_pb.RaftLeadershipTransferRequest{
|
|
TargetId: *targetId,
|
|
TargetAddress: *targetAddress,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("leadership transfer failed: %v", err)
|
|
}
|
|
|
|
if resp.PreviousLeader != resp.NewLeader {
|
|
fmt.Fprintf(writer, "Leadership successfully transferred.\n")
|
|
fmt.Fprintf(writer, " Previous leader: %s\n", resp.PreviousLeader)
|
|
fmt.Fprintf(writer, " New leader: %s\n", resp.NewLeader)
|
|
} else {
|
|
fmt.Fprintf(writer, "Leadership transfer initiated, but the same leader was re-elected.\n")
|
|
fmt.Fprintf(writer, " Current leader: %s\n", resp.NewLeader)
|
|
}
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
fmt.Fprintf(writer, "\nLeadership transfer failed: %v\n", err)
|
|
fmt.Fprintf(writer, "\nTroubleshooting:\n")
|
|
fmt.Fprintf(writer, " - Ensure you are connected to the current leader\n")
|
|
fmt.Fprintf(writer, " - Ensure target server is a voting member (use 'cluster.raft.ps')\n")
|
|
fmt.Fprintf(writer, " - Ensure target server is healthy and reachable\n")
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|