Files
seaweedfs/weed/shell/command_cluster_raft_leader_transfer.go
T
Chris LuandGitHub 902a12fd6f wdclient: bound the wait for a master leader by the caller's context (#11002)
* wdclient: bound the wait for a master leader by the caller's context

WithClient waited on GetMaster with context.Background(), so a caller that
arrived while no master leader was known parked in a 200ms poll loop until one
appeared, whatever deadline it had already set on the RPC. Each retry above it
then left another goroutine in the same wait.

Take the context in WithClient and WithClientCustomGetMaster and hand it to
GetMaster, and stop the retry loop once it is done. The dial keeps
context.Background(): fn brings its own RPC context, so a cancellation seen
here cannot be attributed to the shared connection.

Call sites pass whatever they hold: the request context in the filer's
CollectionList, DeleteCollection and Statistics handlers and in the credential
store's propagation, the operation context in the shell's s3.bucket.delete and
the kafka gateway's broker and filer discovery, and context.Background() where
there is none - the shell commands, the admin dashboard wrapper, and the
exclusive locker's initial lease. The locker's release keeps its own
uncancelled context so a slow unlock cannot turn into a ghost lock.

Claude-Session: https://claude.ai/code/session_01BjDWtZsCoZY6x4pdDmGWxU

* wdclient: test that WithClient gives up with the caller's context

Claude-Session: https://claude.ai/code/session_01BjDWtZsCoZY6x4pdDmGWxU

* wdclient: cut the master retry backoff short when the caller gives up

util.Retry sleeps unconditionally between attempts, so a transient error
arriving just before the caller's deadline still cost it a full backoff step.
Use the context-aware util.RetryWithBackoff, the same helper the volume lookup
in this file already uses.

Two call sites went with it: the shell's lock-holder lookup builds its three
second bound before WithClient so it also covers finding the leader, as its
comment already promised, and the filer's post-delete collection cleanup goes
back to an uncancelled context - the entry is already gone, so a caller that
hung up must not leave the collection behind.

Claude-Session: https://claude.ai/code/session_01BjDWtZsCoZY6x4pdDmGWxU

* wdclient: test that a cancel during backoff ends the retry

Claude-Session: https://claude.ai/code/session_01BjDWtZsCoZY6x4pdDmGWxU
2026-08-27 22:27:45 -07:00

144 lines
4.6 KiB
Go

package shell
import (
"context"
"flag"
"fmt"
"io"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
func init() {
Commands = append(Commands, &commandRaftLeaderTransfer{})
}
type commandRaftLeaderTransfer struct{}
func (c *commandRaftLeaderTransfer) Name() string {
return "cluster.raft.transferLeader"
}
func (c *commandRaftLeaderTransfer) Help() string {
return `transfer raft leadership to another master server
This command initiates a graceful leadership transfer from the current
leader to another server. Use this before performing maintenance on
the current leader to reduce errors in filers and other components.
Examples:
# Transfer to any eligible follower (auto-selection)
cluster.raft.transferLeader
# Transfer to a specific server
cluster.raft.transferLeader -id <server_id> -address <server_grpc_address>
Notes:
- Requires hashicorp raft (-raftHashicorp=true on master)
- This command must be sent to the current leader
- The target server must be a voting member of the raft cluster
- Use 'cluster.raft.ps' to list available servers and identify the leader
`
}
func (c *commandRaftLeaderTransfer) HasTag(CommandTag) bool {
return false
}
func (c *commandRaftLeaderTransfer) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
leaderTransferCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
targetId := leaderTransferCommand.String("id", "", "target server id (must be used with -address)")
targetAddress := leaderTransferCommand.String("address", "", "target server grpc address (must be used with -id)")
if err := leaderTransferCommand.Parse(args); err != nil {
return err
}
// Validate: id and address must be specified together
if *targetId != "" && *targetAddress == "" {
return fmt.Errorf("-address is required when -id is specified")
}
if *targetAddress != "" && *targetId == "" {
return fmt.Errorf("-id is required when -address is specified")
}
// First, show current cluster status
fmt.Fprintf(writer, "Checking current raft cluster status...\n")
var currentLeader string
err := commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.RaftListClusterServers(ctx, &master_pb.RaftListClusterServersRequest{})
if err != nil {
return fmt.Errorf("failed to list cluster servers: %v", err)
}
if len(resp.ClusterServers) == 0 {
fmt.Fprintf(writer, "No raft cluster configured (single master mode)\n")
return fmt.Errorf("leadership transfer not available in single master mode")
}
fmt.Fprintf(writer, "Raft cluster has %d servers:\n", len(resp.ClusterServers))
for _, server := range resp.ClusterServers {
suffix := ""
if server.IsLeader {
suffix = " <- current leader"
currentLeader = server.Id
}
fmt.Fprintf(writer, " %s %s [%s]%s\n", server.Id, server.Address, server.Suffrage, suffix)
}
return nil
})
if err != nil {
return err
}
if currentLeader == "" {
return fmt.Errorf("no leader found in cluster")
}
// Perform the transfer
targetDesc := "any eligible follower"
if *targetId != "" {
targetDesc = fmt.Sprintf("server %s (%s)", *targetId, *targetAddress)
}
fmt.Fprintf(writer, "\nTransferring leadership from %s to %s...\n", currentLeader, targetDesc)
err = commandEnv.MasterClient.WithClient(context.Background(), true, func(client master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := client.RaftLeadershipTransfer(ctx, &master_pb.RaftLeadershipTransferRequest{
TargetId: *targetId,
TargetAddress: *targetAddress,
})
if err != nil {
return fmt.Errorf("leadership transfer failed: %v", err)
}
if resp.PreviousLeader != resp.NewLeader {
fmt.Fprintf(writer, "Leadership successfully transferred.\n")
fmt.Fprintf(writer, " Previous leader: %s\n", resp.PreviousLeader)
fmt.Fprintf(writer, " New leader: %s\n", resp.NewLeader)
} else {
fmt.Fprintf(writer, "Leadership transfer initiated, but the same leader was re-elected.\n")
fmt.Fprintf(writer, " Current leader: %s\n", resp.NewLeader)
}
return nil
})
if err != nil {
fmt.Fprintf(writer, "\nLeadership transfer failed: %v\n", err)
fmt.Fprintf(writer, "\nTroubleshooting:\n")
fmt.Fprintf(writer, " - Ensure you are connected to the current leader\n")
fmt.Fprintf(writer, " - Ensure target server is a voting member (use 'cluster.raft.ps')\n")
fmt.Fprintf(writer, " - Ensure target server is healthy and reachable\n")
return err
}
return nil
}