server: 416 only when no requested range overlaps, with Content-Range, and the Rust mirror (#10889)

* filer, volume server: return 416 when no requested range overlaps the content

* seaweed-volume: return 416 when no requested range overlaps the content

* server: check the range test error, use the request context, fix the no-overlap comment boundary

seaweed-volume/src/server/handlers.rs

@@ -1589,12 +1589,15 @@ async fn get_or_head_handler_inner(
 }
 
 /// Handle HTTP Range requests. Returns 206 Partial Content or 416 Range Not Satisfiable.
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug)]
 struct HttpRange {
     start: i64,
     length: i64,
 }
 
+// Returned when the first-byte-pos of every byte-range-spec is at or past the content size.
+const RANGE_NO_OVERLAP: &str = "invalid range: failed to overlap";
+
 fn parse_range_header(s: &str, size: i64) -> Result<Vec<HttpRange>, &'static str> {
     if s.is_empty() {
         return Ok(Vec::new());
@@ -1604,6 +1607,7 @@ fn parse_range_header(s: &str, size: i64) -> Result<Vec<HttpRange>, &'static str
         return Err("invalid range");
     }
     let mut ranges = Vec::new();
+    let mut no_overlap = false;
     for part in s[PREFIX.len()..].split(',') {
         let part = part.trim();
         if part.is_empty() {
@@ -1627,9 +1631,13 @@ fn parse_range_header(s: &str, size: i64) -> Result<Vec<HttpRange>, &'static str
             r.length = size - r.start;
         } else {
             let i = start_str.parse::<i64>().map_err(|_| "invalid range")?;
-            if i > size || i < 0 {
+            if i < 0 {
                 return Err("invalid range");
             }
+            if i >= size {
+                no_overlap = true;
+                continue;
+            }
             r.start = i;
             if end_str.is_empty() {
                 r.length = size - r.start;
@@ -1646,6 +1654,9 @@ fn parse_range_header(s: &str, size: i64) -> Result<Vec<HttpRange>, &'static str
         }
         ranges.push(r);
     }
+    if no_overlap && ranges.is_empty() {
+        return Err(RANGE_NO_OVERLAP);
+    }
     Ok(ranges)
 }
 
@@ -1679,7 +1690,15 @@ fn handle_range_request(
     let total = data.len() as i64;
     let ranges = match parse_range_header(range_str, total) {
         Ok(r) => r,
-        Err(msg) => return range_error_response(headers, msg),
+        Err(msg) => {
+            if msg == RANGE_NO_OVERLAP {
+                headers.insert(
+                    "Content-Range",
+                    format!("bytes */{}", total).parse().unwrap(),
+                );
+            }
+            return range_error_response(headers, msg);
+        }
     };
 
     // Go's ProcessRangeRequest returns nil (empty body) for empty or oversized ranges
@@ -1762,7 +1781,15 @@ fn handle_range_request_from_source(
     let total = info.data_size as i64;
     let ranges = match parse_range_header(range_str, total) {
         Ok(r) => r,
-        Err(msg) => return range_error_response(headers, msg),
+        Err(msg) => {
+            if msg == RANGE_NO_OVERLAP {
+                headers.insert(
+                    "Content-Range",
+                    format!("bytes */{}", total).parse().unwrap(),
+                );
+            }
+            return range_error_response(headers, msg);
+        }
     };
 
     if ranges.is_empty() {
@@ -3912,6 +3939,25 @@ mod tests {
         assert!(parse_url_path("").is_none());
     }
 
+    #[test]
+    fn test_parse_range_header_no_overlap() {
+        assert_eq!(
+            parse_range_header("bytes=10-", 10).unwrap_err(),
+            RANGE_NO_OVERLAP
+        );
+        assert_eq!(
+            parse_range_header("bytes=100-", 10).unwrap_err(),
+            RANGE_NO_OVERLAP
+        );
+        // 416 only when every range fails to overlap
+        let ranges = parse_range_header("bytes=10-,0-1", 10).unwrap();
+        assert_eq!(ranges.len(), 1);
+        assert_eq!((ranges[0].start, ranges[0].length), (0, 2));
+        // an end past the size is clamped, still satisfiable
+        let ranges = parse_range_header("bytes=5-100", 10).unwrap();
+        assert_eq!((ranges[0].start, ranges[0].length), (5, 5));
+    }
+
     #[test]
     fn test_extract_jwt_bearer() {
         let mut headers = HeaderMap::new();

weed/server/common.go

@@ -316,6 +316,9 @@ func ProcessRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
 	ranges, err := parseRange(rangeReq, totalSize)
 	if err != nil {
 		glog.Errorf("ProcessRangeRequest headers: %+v err: %v", w.Header(), err)
+		if err == errNoOverlap {
+			w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", totalSize))
+		}
 		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
 		return fmt.Errorf("ProcessRangeRequest header: %w", err)
 	}

weed/server/common_test.go

@@ -8,6 +8,8 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
 )
 
 func TestParseURL(t *testing.T) {
@@ -112,3 +114,47 @@ func (c *countingReadCloser) Read(p []byte) (int, error) {
 func (c *countingReadCloser) Close() error {
 	return nil
 }
+
+func TestProcessRangeRequestRanges(t *testing.T) {
+	data := []byte("0123456789")
+	serve := func(rangeHeader string) (*httptest.ResponseRecorder, error) {
+		r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/test.txt", nil)
+		r.Header.Set("Range", rangeHeader)
+		w := httptest.NewRecorder()
+		err := ProcessRangeRequest(r, w, int64(len(data)), "text/plain", func(offset int64, size int64) (filer.DoStreamContent, error) {
+			return func(writer io.Writer) error {
+				_, err := writer.Write(data[offset : offset+size])
+				return err
+			}, nil
+		})
+		return w, err
+	}
+
+	tests := []struct {
+		rangeHeader string
+		wantCode    int
+		wantRange   string
+		wantBody    string
+	}{
+		{"bytes=0-1", http.StatusPartialContent, "bytes 0-1/10", "01"},
+		{"bytes=5-100", http.StatusPartialContent, "bytes 5-9/10", "56789"},
+		{"bytes=10-", http.StatusRequestedRangeNotSatisfiable, "bytes */10", ""},
+		{"bytes=100-", http.StatusRequestedRangeNotSatisfiable, "bytes */10", ""},
+		{"bytes=10-,0-1", http.StatusPartialContent, "bytes 0-1/10", "01"},
+	}
+	for _, tt := range tests {
+		w, err := serve(tt.rangeHeader)
+		if wantErr := tt.wantCode == http.StatusRequestedRangeNotSatisfiable; (err != nil) != wantErr {
+			t.Errorf("%s: error = %v, want an error only for 416", tt.rangeHeader, err)
+		}
+		if w.Code != tt.wantCode {
+			t.Errorf("%s: status %d, want %d", tt.rangeHeader, w.Code, tt.wantCode)
+		}
+		if got := w.Header().Get("Content-Range"); got != tt.wantRange {
+			t.Errorf("%s: Content-Range %q, want %q", tt.rangeHeader, got, tt.wantRange)
+		}
+		if tt.wantBody != "" && w.Body.String() != tt.wantBody {
+			t.Errorf("%s: body %q, want %q", tt.rangeHeader, w.Body.String(), tt.wantBody)
+		}
+	}
+}

weed/server/volume_server_handlers_helper.go

@@ -27,7 +27,11 @@ func (r httpRange) mimeHeader(contentType string, size int64) textproto.MIMEHead
 	}
 }
 
-// parseRange parses a Range header string as per RFC 2616.
+// errNoOverlap is returned by parseRange if first-byte-pos of
+// all of the byte-range-spec values is at or beyond the content size.
+var errNoOverlap = errors.New("invalid range: failed to overlap")
+
+// parseRange parses a Range header string as per RFC 7233.
 func parseRange(s string, size int64) ([]httpRange, error) {
 	if s == "" {
 		return nil, nil // header not present
@@ -37,6 +41,7 @@ func parseRange(s string, size int64) ([]httpRange, error) {
 		return nil, errors.New("invalid range")
 	}
 	var ranges []httpRange
+	noOverlap := false
 	for _, ra := range strings.Split(s[len(b):], ",") {
 		ra = strings.TrimSpace(ra)
 		if ra == "" {
@@ -62,9 +67,15 @@ func parseRange(s string, size int64) ([]httpRange, error) {
 			r.length = size - r.start
 		} else {
 			i, err := strconv.ParseInt(start, 10, 64)
-			if err != nil || i >= size || i < 0 {
+			if err != nil || i < 0 {
 				return nil, errors.New("invalid range")
 			}
+			if i >= size {
+				// If the range begins after the size of the content,
+				// then it does not overlap.
+				noOverlap = true
+				continue
+			}
 			r.start = i
 			if end == "" {
 				// If no end is specified, range extends to end of the file.
@@ -82,6 +93,10 @@ func parseRange(s string, size int64) ([]httpRange, error) {
 		}
 		ranges = append(ranges, r)
 	}
+	if noOverlap && len(ranges) == 0 {
+		// The specified ranges did not overlap with the content.
+		return nil, errNoOverlap
+	}
 	return ranges, nil
 }