From 3441a2a7f13123abf63d1249d5c790f3994116ed Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 30 May 2026 15:07:27 -0700 Subject: [PATCH] s3: short-circuit filer failover on ErrNotFound (#9748) withFilerClientFailover treated a filer's ErrNotFound like a transport failure: it kept the result, re-queried every other filer, and finally wrapped the answer as "all filers failed, last error: ... no entry is found in filer store". For workloads with many legitimate misses (e.g. GET object?versionId=X for a version that was deleted or expired), this turned each 404 into N filer round-trips and produced a misleading error string. A reachable filer that answers ErrNotFound has given an authoritative answer; failover exists to route around unreachable or unhealthy filers, not to look harder for an entry the store reports as absent. Return ErrNotFound directly instead of fanning out. Callers that need read-after-write retries already handle that at the S3 semantic layer (e.g. getLatestObjectVersion). --- weed/s3api/s3api_handlers.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index 7f74e5dd4..a1f46795c 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -47,14 +47,14 @@ func (s3a *S3ApiServer) withFilerClientFailover(streamingMode bool, fn func(file return nil } - // ErrNotFound is a valid application-level response (entry doesn't exist on this filer), - // not a filer health issue. Only record true failures (transport errors, timeouts, etc.) - // in the health tracker to avoid poisoning the circuit breaker with normal "not found" - // responses in multi-filer setups. - if !errors.Is(err, filer_pb.ErrNotFound) { - s3a.filerClient.RecordFilerFailure(currentFiler) + // A reachable filer answering ErrNotFound is authoritative; failover is for + // unreachable/unhealthy filers, not for re-asking about an absent entry. + if errors.Is(err, filer_pb.ErrNotFound) { + return err } + s3a.filerClient.RecordFilerFailure(currentFiler) + // Current filer failed - try all other filers with health-aware selection filers := s3a.filerClient.GetAllFilers() var lastErr error = err @@ -83,10 +83,12 @@ func (s3a *S3ApiServer) withFilerClientFailover(streamingMode bool, fn func(file return nil } - // Only record real failures, not ErrNotFound - if !errors.Is(err, filer_pb.ErrNotFound) { - s3a.filerClient.RecordFilerFailure(filer) + // Authoritative not-found - stop failing over. + if errors.Is(err, filer_pb.ErrNotFound) { + return err } + + s3a.filerClient.RecordFilerFailure(filer) glog.V(2).Infof("WithFilerClient: failover to %s failed: %v", filer, err) lastErr = err }