s3api: optimize encodePath memory allocations (#10252)

* Optimize s3api encodePath to eliminate O(n^2) allocations encodePath built its result via string concatenation in a loop, which is O(n^2) in allocations. For non-ASCII (e.g. Chinese) runes it additionally called make + hex.EncodeToString + strings.ToUpper per byte (~10 allocations per character). Under high QPS with long non-ASCII object keys this produced a very high allocation rate, frequent GC and long GC pauses, causing S3 request latency spikes. Replace with a preallocated strings.Builder, a manual hex lookup table, and a zero-allocation fast path. EncodePath now delegates to encodePath to remove the duplicated implementation. Output is byte-for-byte identical, verified by TestEncodePath and TestEncodePathEqual. Benchmark (long Chinese path): 261 -> 1 allocs/op, 35810 -> 480 B/op, 7.5x faster. Load test (50M calls): 212x fewer allocations, 76x fewer GC cycles. * s3api: drop regexp from encodePath fast path The unreserved-character scan already decides whether any byte needs encoding, so reservedObjectNames.MatchString was a redundant second pass that also ran the RE2 engine on every authenticated request. Rely on the scan alone; output is unchanged. The ASCII fast path drops from ~188ns to ~11ns per call. --------- Co-authored-by: LiuDoge <liudoge@LiuDogedeMacBook-Air.local> Co-authored-by: Chris Lu <chris.lu@gmail.com>
2026-09-03 14:47:04 +00:00 · 2026-07-07 12:36:47 -07:00
parent 4f1f0dcb17
commit f58721b22f
2 changed files with 162 additions and 19 deletions
@@ -1076,6 +1076,19 @@ func getSignedHeaders(signedHeaders http.Header) string {
 // if object matches reserved string, no need to encode them
 var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")

+// pathHexTable is used for manual percent-encoding in encodePath to avoid
+// the allocations of hex.EncodeToString + strings.ToUpper.
+const pathHexTable = "0123456789ABCDEF"
+
+// isPathUnreservedChar reports whether s is an RFC 3986 §2.3 unreserved
+// character (or '/') that does not need percent-encoding in an object path.
+func isPathUnreservedChar(s rune) bool {
+	return 'A' <= s && s <= 'Z' ||
+		'a' <= s && s <= 'z' ||
+		'0' <= s && s <= '9' ||
+		s == '-' || s == '_' || s == '.' || s == '~' || s == '/'
+}
+
 // encodePath encodes the strings from UTF-8 byte representations to HTML hex escape sequences
 //
 // This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
@@ -1084,32 +1097,42 @@ var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
 // This function on the other hand is a direct replacement for url.Encode() technique to support
 // pretty much every UTF-8 character.
 func encodePath(pathName string) string {
-	if reservedObjectNames.MatchString(pathName) {
+	// Fast path: if every character is unreserved, the encoded form equals the
+	// input, so return it unchanged with zero allocation. This is the common
+	// case for ASCII object keys and avoids the regexp engine on the SigV4 hot
+	// path, where encodePath runs on every authenticated request.
+	needEncode := false
+	for _, s := range pathName {
+		if !isPathUnreservedChar(s) {
+			needEncode = true
+			break
+		}
+	}
+	if !needEncode {
 		return pathName
 	}
-	var encodedPathname string
+
+	// Slow path: preallocated Builder + manual hex encoding.
+	var buf strings.Builder
+	buf.Grow(len(pathName) * 3) // encoded form is at most 3x the byte length
 	for _, s := range pathName {
-		if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
-			encodedPathname = encodedPathname + string(s)
+		if isPathUnreservedChar(s) { // §2.3 Unreserved characters (mark)
+			buf.WriteRune(s)
 		} else {
-			switch s {
-			case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
-				encodedPathname = encodedPathname + string(s)
-			default:
-				runeLen := utf8.RuneLen(s)
-				if runeLen < 0 {
-					return pathName
-				}
-				u := make([]byte, runeLen)
-				utf8.EncodeRune(u, s)
-				for _, r := range u {
-					hex := hex.EncodeToString([]byte{r})
-					encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
-				}
+			runeLen := utf8.RuneLen(s)
+			if runeLen < 0 {
+				return pathName
+			}
+			u := make([]byte, runeLen)
+			utf8.EncodeRune(u, s)
+			for _, r := range u {
+				buf.WriteByte('%')
+				buf.WriteByte(pathHexTable[r>>4])
+				buf.WriteByte(pathHexTable[r&0x0f])
 			}
 		}
 	}
-	return encodedPathname
+	return buf.String()
 }

 // getSignature final signature in hexadecimal form.
@@ -0,0 +1,120 @@
+package s3api
+
+import (
+	"encoding/hex"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+// encodePathOld is the original implementation, kept only in tests as a
+// reference to prove the optimized encodePath produces identical output.
+func encodePathOld(pathName string) string {
+	if reservedObjectNames.MatchString(pathName) {
+		return pathName
+	}
+	var encodedPathname string
+	for _, s := range pathName {
+		if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' {
+			encodedPathname = encodedPathname + string(s)
+		} else {
+			switch s {
+			case '-', '_', '.', '~', '/':
+				encodedPathname = encodedPathname + string(s)
+			default:
+				runeLen := utf8.RuneLen(s)
+				if runeLen < 0 {
+					return pathName
+				}
+				u := make([]byte, runeLen)
+				utf8.EncodeRune(u, s)
+				for _, r := range u {
+					h := hex.EncodeToString([]byte{r})
+					encodedPathname = encodedPathname + "%" + strings.ToUpper(h)
+				}
+			}
+		}
+	}
+	return encodedPathname
+}
+
+// TestEncodePath checks encodePath against explicit expected outputs.
+func TestEncodePath(t *testing.T) {
+	cases := []struct {
+		in   string
+		want string
+	}{
+		{"/bucket/file.txt", "/bucket/file.txt"},
+		{"/a-b_c.d~e/f", "/a-b_c.d~e/f"},
+		{"", ""},
+		{"/", "/"},
+		{"/中", "/%E4%B8%AD"},
+		{"/a b", "/a%20b"},
+		{"/a&b=c", "/a%26b%3Dc"},
+	}
+	for _, c := range cases {
+		if got := encodePath(c.in); got != c.want {
+			t.Errorf("encodePath(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+// TestEncodePathEqual verifies the optimized encodePath produces byte-for-byte
+// identical output to the original implementation across many inputs.
+func TestEncodePathEqual(t *testing.T) {
+	cases := []string{
+		"/bucket/file.txt",
+		"/my-bucket/data/2026/07/file.parquet",
+		"/我的存储桶/数据仓库/文件.数据",
+		"/bucket/项目数据/report-分析.parquet",
+		"/path with spaces/and&symbols=test/文件.txt",
+		"/emoji/😀/file.txt", // 4-byte UTF-8 rune
+		"/",
+		"/a",
+		"",
+		"/纯中文路径没有斜杠结尾/文件名",
+		"/mixed混合/path路径/2026年/data.csv",
+	}
+	for _, p := range cases {
+		if got, want := encodePath(p), encodePathOld(p); got != want {
+			t.Errorf("mismatch for %q:\n  optimized=%q\n  original =%q", p, got, want)
+		}
+	}
+}
+
+// benchPaths covers different path types for benchmarking.
+var benchPaths = []string{
+	"/bucket/file.txt", // short ASCII
+	"/my-bucket/data/2026/07/07/service/module/submodule/very/long/path/to/object/file-name-1234567890.parquet", // long ASCII
+	"/我的存储桶/数据仓库/2026年07月/业务数据/用户行为分析/长长的中文文件路径/这是一个很长的中文对象名称文件.数据",                                             // long non-ASCII
+	"/bucket/项目数据/2026/报表/月度统计/user-behavior-分析报告-统计数据-长文件名称-1234567890.parquet",                                // mixed
+}
+
+func benchLabel(p string) string {
+	if p == "" {
+		return "<empty>"
+	}
+	if len(p) > 20 {
+		return p[:20] + "..."
+	}
+	return p
+}
+
+// BenchmarkEncodePath benchmarks the original vs optimized implementation.
+func BenchmarkEncodePath(b *testing.B) {
+	for _, p := range benchPaths {
+		label := benchLabel(p)
+		b.Run("Old/"+label, func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				_ = encodePathOld(p)
+			}
+		})
+		b.Run("New/"+label, func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				_ = encodePath(p)
+			}
+		})
+	}
+}