diff --git a/weed/server/common.go b/weed/server/common.go index 2d3c55466..65a09c280 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -3,6 +3,7 @@ package weed_server import ( "bufio" "bytes" + "context" "encoding/json" "errors" "fmt" @@ -28,6 +29,7 @@ import ( "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" ) @@ -39,6 +41,35 @@ var writePool = sync.Pool{New: func() interface{} { }, } +// ErrCacheNotReady signals that a remote-only object's local cache is still +// filling. Callers should map it to 503 + Retry-After so SDKs back off and retry. +var ErrCacheNotReady = errors.New("remote object not cached yet") + +// writePrepareWriteFnErr writes an HTTP response for an error from +// prepareWriteFn, before any 2xx headers have been written. Client cancels are +// silent; filer_pb.ErrNotFound becomes 404; ErrCacheNotReady becomes 503 + +// Retry-After; everything else is 500. Strips headers that described the +// success body (Content-Length / Content-Range / Content-Disposition / ETag / +// Last-Modified) so they don't get attached to the error response. +func writePrepareWriteFnErr(w http.ResponseWriter, err error) { + for _, h := range []string{"Content-Length", "Content-Range", "Content-Disposition", "ETag", "Last-Modified"} { + w.Header().Del(h) + } + switch { + case errors.Is(err, context.Canceled): + glog.V(3).Infof("ProcessRangeRequest: client disconnected: %v", err) + case errors.Is(err, filer_pb.ErrNotFound): + http.Error(w, err.Error(), http.StatusNotFound) + case errors.Is(err, ErrCacheNotReady): + glog.V(1).Infof("ProcessRangeRequest: cache not ready, returning 503: %v", err) + w.Header().Set("Retry-After", "5") + http.Error(w, err.Error(), http.StatusServiceUnavailable) + default: + glog.Errorf("ProcessRangeRequest: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + func init() { serverStats = stats.NewServerStats() go serverStats.Start() @@ -290,9 +321,7 @@ func ProcessRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) writeFn, err := prepareWriteFn(0, totalSize) if err != nil { - glog.Errorf("ProcessRangeRequest: %v", err) - w.Header().Del("Content-Length") - http.Error(w, err.Error(), http.StatusInternalServerError) + writePrepareWriteFnErr(w, err) return fmt.Errorf("ProcessRangeRequest: %w", err) } if err = writeFn(bufferedWriter); err != nil { @@ -340,9 +369,7 @@ func ProcessRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 writeFn, err := prepareWriteFn(ra.start, ra.length) if err != nil { - glog.Errorf("ProcessRangeRequest range[0]: %+v err: %v", w.Header(), err) - w.Header().Del("Content-Length") - http.Error(w, err.Error(), http.StatusInternalServerError) + writePrepareWriteFnErr(w, err) return fmt.Errorf("ProcessRangeRequest: %w", err) } w.WriteHeader(http.StatusPartialContent) @@ -365,9 +392,8 @@ func ProcessRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 } writeFn, err := prepareWriteFn(ra.start, ra.length) if err != nil { - glog.Errorf("ProcessRangeRequest range[%d] err: %v", i, err) - http.Error(w, "Internal Error", http.StatusInternalServerError) - return fmt.Errorf("ProcessRangeRequest range[%d] err: %v", i, err) + writePrepareWriteFnErr(w, err) + return fmt.Errorf("ProcessRangeRequest range[%d]: %w", i, err) } writeFnByRange[i] = writeFn } diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 61a87cd97..234bc264b 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -2,6 +2,7 @@ package weed_server import ( "context" + "errors" "fmt" "io" "math" @@ -211,8 +212,18 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) Name: name, }); err != nil { stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadCache).Inc() - glog.ErrorfCtx(ctx, "CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err) - return nil, fmt.Errorf("cache %s: %v", entry.FullPath, err) + // Client disconnected: surface ctx error so caller stays silent. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + // Entry vanished mid-cache: forward NotFound so caller maps to 404, + // not the 503 retry-loop. + if errors.Is(err, filer_pb.ErrNotFound) { + return nil, err + } + // Cache still filling: tag with sentinel so caller maps to 503 + Retry-After. + glog.WarningfCtx(ctx, "CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err) + return nil, fmt.Errorf("cache %s: %w", entry.FullPath, ErrCacheNotReady) } else { chunks = resp.Entry.GetChunks() }