diff --git a/backend/_example/memory_store/go.mod b/backend/_example/memory_store/go.mod index 8f66d9f2..0b7cbb3a 100644 --- a/backend/_example/memory_store/go.mod +++ b/backend/_example/memory_store/go.mod @@ -3,7 +3,7 @@ module github.com/umputun/remark42/memory_store go 1.25.0 require ( - github.com/go-pkgz/jrpc v0.4.0 + github.com/go-pkgz/jrpc v0.4.2 github.com/go-pkgz/lgr v0.12.4 github.com/jessevdk/go-flags v1.6.1 github.com/stretchr/testify v1.12.0 @@ -16,9 +16,9 @@ require ( github.com/alecthomas/chroma/v2 v2.27.0 // indirect github.com/andybalholm/cascadia v1.3.4 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/dlclark/regexp2/v2 v2.2.2 // indirect + github.com/dlclark/regexp2/v2 v2.7.1 // indirect github.com/go-pkgz/rest v1.24.0 // indirect - github.com/go-pkgz/routegroup v1.6.0 // indirect + github.com/go-pkgz/routegroup v1.6.1 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect @@ -28,7 +28,7 @@ require ( go.etcd.io/bbolt v1.5.0 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/image v0.45.0 // indirect - golang.org/x/net v0.57.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.47.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/backend/_example/memory_store/go.sum b/backend/_example/memory_store/go.sum index f1f61867..794132b4 100644 --- a/backend/_example/memory_store/go.sum +++ b/backend/_example/memory_store/go.sum @@ -13,16 +13,16 @@ github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0= -github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= -github.com/go-pkgz/jrpc v0.4.0 h1:oD7xiGrzDkndkuCjeHGugQXxbggLSV7O1QmHhoc5pYY= -github.com/go-pkgz/jrpc v0.4.0/go.mod h1:JFoY3bRjRyx4M3CbEVDFQStMB1m2gmQ7OjqFK7q3kOo= +github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4= +github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= +github.com/go-pkgz/jrpc v0.4.2 h1:gY5mmxp9/dFd1WsHybVZILQpF11YNWWS3Ga+Pc5aIAU= +github.com/go-pkgz/jrpc v0.4.2/go.mod h1:ZtnMpIXYmwXh6W44XO2lE5Lh5J+6KeeMIvw+vF9xXRQ= github.com/go-pkgz/lgr v0.12.4 h1:lDeQ4BR28ldXrKau6BOjq7A8nHzcXz+MF4xUfV4l1Ok= github.com/go-pkgz/lgr v0.12.4/go.mod h1:Lw6DkNRnCPyX07mqkiUK/p+eA1opq4GKkWfWia64RA8= github.com/go-pkgz/rest v1.24.0 h1:GAUCgx7U8xCOC2OynLjhCRMhtnMQH4d1mTdKpQyX2yI= github.com/go-pkgz/rest v1.24.0/go.mod h1:dl3EWiuFB4hRTo2Sknj6UrQGFRAYvANK6/NyW8qQPxc= -github.com/go-pkgz/routegroup v1.6.0 h1:44XHZgF6JIIldRlv+zjg6SygULASmjifnfIQjwCT0e4= -github.com/go-pkgz/routegroup v1.6.0/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg= +github.com/go-pkgz/routegroup v1.6.1 h1:6I/0LabazpZsHAI+jYPeyH/KU2cvZF0bFylUScMNi+Q= +github.com/go-pkgz/routegroup v1.6.1/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -49,8 +49,8 @@ golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= diff --git a/backend/go.mod b/backend/go.mod index 27a93c97..edac13a3 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -8,19 +8,19 @@ require ( github.com/alecthomas/chroma/v2 v2.27.0 github.com/didip/tollbooth/v8 v8.0.1 github.com/go-pkgz/auth/v2 v2.2.0 - github.com/go-pkgz/jrpc v0.4.0 - github.com/go-pkgz/lcw/v2 v2.0.0 + github.com/go-pkgz/jrpc v0.4.2 + github.com/go-pkgz/lcw/v2 v2.1.0 github.com/go-pkgz/lgr v0.12.4 github.com/go-pkgz/notify v1.4.0 github.com/go-pkgz/repeater/v2 v2.2.0 github.com/go-pkgz/rest v1.24.0 - github.com/go-pkgz/routegroup v1.6.0 - github.com/go-pkgz/syncs v1.3.2 + github.com/go-pkgz/routegroup v1.6.1 + github.com/go-pkgz/syncs v1.3.3 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/gorilla/feeds v1.2.0 github.com/jessevdk/go-flags v1.6.1 - github.com/kyokomi/emoji/v2 v2.2.13 + github.com/kyokomi/emoji/v2 v2.2.14 github.com/microcosm-cc/bluemonday v1.0.27 github.com/rs/xid v1.6.0 github.com/russross/blackfriday/v2 v2.1.0 @@ -30,7 +30,7 @@ require ( go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.55.0 golang.org/x/image v0.45.0 - golang.org/x/net v0.57.0 + golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 ) @@ -40,21 +40,19 @@ require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dghubble/oauth1 v0.7.3 // indirect - github.com/dlclark/regexp2/v2 v2.2.2 // indirect + github.com/dlclark/regexp2/v2 v2.7.1 // indirect github.com/go-oauth2/oauth2/v4 v4.5.4 // indirect github.com/go-pkgz/email v0.8.0 // indirect - github.com/go-pkgz/expirable-cache/v3 v3.1.0 // indirect + github.com/go-pkgz/expirable-cache/v3 v3.1.1 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/montanaflynn/stats v0.12.4 // indirect - github.com/redis/go-redis/v9 v9.21.0 // indirect + github.com/redis/go-redis/v9 v9.22.0 // indirect github.com/rrivera/identicon v0.0.0-20240116195454-d5ba35832c0d // indirect - github.com/slack-go/slack v0.27.0 // indirect + github.com/slack-go/slack v0.29.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect diff --git a/backend/go.sum b/backend/go.sum index 646a7d2a..3c4f0d0a 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -12,10 +12,8 @@ github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= -github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y= -github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg= +github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw= +github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg= @@ -34,8 +32,8 @@ github.com/dghubble/oauth1 v0.7.3 h1:EkEM/zMDMp3zOsX2DC/ZQ2vnEX3ELK0/l9kb+vs4ptE github.com/dghubble/oauth1 v0.7.3/go.mod h1:oxTe+az9NSMIucDPDCCtzJGsPhciJV33xocHfcR2sVY= github.com/didip/tollbooth/v8 v8.0.1 h1:VAAapTo1t4Bn6bbpcHjuovwoa9u3JH++wgjbpWv+rB8= github.com/didip/tollbooth/v8 v8.0.1/go.mod h1:oEd9l+ep373d7DmvKLc0a5gasPOev2mTewi6KPQBGJ4= -github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0= -github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= +github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4= +github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/gavv/httpexpect v2.0.0+incompatible h1:1X9kcRshkSKEjNJJxX9Y9mQ5BRfbxU5kORdjhlA1yX8= @@ -46,12 +44,12 @@ github.com/go-pkgz/auth/v2 v2.2.0 h1:vQO+GTFDjAaBNSdcLLLr3Xibka67GZPtkRRItVVS4ow github.com/go-pkgz/auth/v2 v2.2.0/go.mod h1:iZx2JiGZ8Aef+wM0BPLMQY8aur4fLE0uyPhFRO9dYQ4= github.com/go-pkgz/email v0.8.0 h1:6+Tgjfj7zFccFCPmURV2spKDXDb7aX/iWXBY2hBa9ww= github.com/go-pkgz/email v0.8.0/go.mod h1:+wgi4x7S33IuCzfcCM5euN0GwQG6XvO/PBLxrNffYLI= -github.com/go-pkgz/expirable-cache/v3 v3.1.0 h1:s05P851/O6QJ6Mc+7o2bh9aGtD3romB1SxDTXifdoqc= -github.com/go-pkgz/expirable-cache/v3 v3.1.0/go.mod h1:6pVgNleydKPj0J2/mzrI02/RDo4ivKx5v2XlNmIjhjo= -github.com/go-pkgz/jrpc v0.4.0 h1:oD7xiGrzDkndkuCjeHGugQXxbggLSV7O1QmHhoc5pYY= -github.com/go-pkgz/jrpc v0.4.0/go.mod h1:JFoY3bRjRyx4M3CbEVDFQStMB1m2gmQ7OjqFK7q3kOo= -github.com/go-pkgz/lcw/v2 v2.0.0 h1:gTwXpiJBhQeA1rXuqkRuLcV79uATFna8CckH8ZBBrH0= -github.com/go-pkgz/lcw/v2 v2.0.0/go.mod h1:yxJHOn+IbQBQHxUqkCtMrbGjIfdYcsBAZcVCBaL1Va8= +github.com/go-pkgz/expirable-cache/v3 v3.1.1 h1:ryHiSI5gBE8aJ0Jt90VFMKZxe/cosf2dZPDcUTMaHNg= +github.com/go-pkgz/expirable-cache/v3 v3.1.1/go.mod h1:peJAuIDjP76Uuc9NK55ljQlBtwwmJDvx4CnMyUcsP40= +github.com/go-pkgz/jrpc v0.4.2 h1:gY5mmxp9/dFd1WsHybVZILQpF11YNWWS3Ga+Pc5aIAU= +github.com/go-pkgz/jrpc v0.4.2/go.mod h1:ZtnMpIXYmwXh6W44XO2lE5Lh5J+6KeeMIvw+vF9xXRQ= +github.com/go-pkgz/lcw/v2 v2.1.0 h1:JAGUHRQPon658XimxIUwaqPgOPfIKa850qO8jpFJchc= +github.com/go-pkgz/lcw/v2 v2.1.0/go.mod h1:UUo4cgD6oTPooBuUslVaWqOYZAGnh/91SoaB5DBWC8s= github.com/go-pkgz/lgr v0.12.4 h1:lDeQ4BR28ldXrKau6BOjq7A8nHzcXz+MF4xUfV4l1Ok= github.com/go-pkgz/lgr v0.12.4/go.mod h1:Lw6DkNRnCPyX07mqkiUK/p+eA1opq4GKkWfWia64RA8= github.com/go-pkgz/notify v1.4.0 h1:4pP7UGdYqFO7e7V3OsQStYF006CO0cCh1ahdawt6l18= @@ -60,10 +58,10 @@ github.com/go-pkgz/repeater/v2 v2.2.0 h1:8nZR/NaknmLfx2YMHbr78u9OL4Xj+8+romm9dz4 github.com/go-pkgz/repeater/v2 v2.2.0/go.mod h1:RgX5vUbLKq7PV82QUDP5pFbQS1os4Z+U9XzKymK23A8= github.com/go-pkgz/rest v1.24.0 h1:GAUCgx7U8xCOC2OynLjhCRMhtnMQH4d1mTdKpQyX2yI= github.com/go-pkgz/rest v1.24.0/go.mod h1:dl3EWiuFB4hRTo2Sknj6UrQGFRAYvANK6/NyW8qQPxc= -github.com/go-pkgz/routegroup v1.6.0 h1:44XHZgF6JIIldRlv+zjg6SygULASmjifnfIQjwCT0e4= -github.com/go-pkgz/routegroup v1.6.0/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg= -github.com/go-pkgz/syncs v1.3.2 h1:gmioASlJNy3gNosPlgvWOM2QP0Hdjzn2u+/sUShgd8E= -github.com/go-pkgz/syncs v1.3.2/go.mod h1:qjgzpp7OpuhDf7BWsW/FHCu9DLjE32NPy6/vXAXT/Cw= +github.com/go-pkgz/routegroup v1.6.1 h1:6I/0LabazpZsHAI+jYPeyH/KU2cvZF0bFylUScMNi+Q= +github.com/go-pkgz/routegroup v1.6.1/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg= +github.com/go-pkgz/syncs v1.3.3 h1:fFRK+eqCIFddxEiDi6ob5oQh3/NuNkjuwWQJzZ0b9lU= +github.com/go-pkgz/syncs v1.3.3/go.mod h1:lAp+w+qbRm6+pwINcfc9BVK+4F4NEoERYditsF1uENA= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -84,11 +82,6 @@ github.com/gorilla/feeds v1.2.0 h1:O6pBiXJ5JHhPvqy53NsjKOThq+dNFm8+DFrxBEdzSCc= github.com/gorilla/feeds v1.2.0/go.mod h1:WMib8uJP3BbY+X8Szd1rA5Pzhdfh+HCCAYT2z7Fza6Y= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -107,16 +100,16 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO7U= -github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= +github.com/kyokomi/emoji/v2 v2.2.14 h1:YOF6VL52613M0Qr9v4puJDD9QQPmyyjXedDDlrGzH80= +github.com/kyokomi/emoji/v2 v2.2.14/go.mod h1:1AnYl9IgmJZXKd5m1PEijyyUw85SqYsuAr8lpU/s+9s= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/montanaflynn/stats v0.12.4 h1:amtNRsti20yIhcrkfUJGwoYqBR82jKQFE8SNNYVgGn0= github.com/montanaflynn/stats v0.12.4/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/redis/go-redis/v9 v9.21.0 h1:FPBE4hhbAke+TLmcY3WkpbDffJEomdqPn3HYiqAtL9E= -github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rrivera/identicon v0.0.0-20240116195454-d5ba35832c0d h1:l3+2LWCbVxn5itfvXAfH9n4YL9jh8l1g5zcncbIc1cs= @@ -129,8 +122,8 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/slack-go/slack v0.27.0 h1:VWOpUzOK6UAPCCQlFxl79jhv8a/b+GOSJMnWziDJ8B8= -github.com/slack-go/slack v0.27.0/go.mod h1:UEe+jmo9WLlwHB04qsOrTDvqM7Aa4rQL3O5wF3n0hx4= +github.com/slack-go/slack v0.29.0 h1:ohhMNgp9DmPKiLhH/pNZV4NxhOXKgNy0SH8FzVHNerI= +github.com/slack-go/slack v0.29.0/go.mod h1:UEe+jmo9WLlwHB04qsOrTDvqM7Aa4rQL3O5wF3n0hx4= github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -200,8 +193,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/README.md b/backend/vendor/github.com/dlclark/regexp2/v2/README.md index f49a0d05..0095bf40 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/README.md +++ b/backend/vendor/github.com/dlclark/regexp2/v2/README.md @@ -38,7 +38,7 @@ if isMatch, _ := re.MatchString(`Something to match`); isMatch { } ``` -The only error that the `*Match*` methods *should* return is a Timeout if you set the `re.MatchTimeout` field. Any other error is a bug in the `regexp2` package. If you need more details about capture groups in a match then use the `FindStringMatch` method, like so: +The `*Match*` methods can return a timeout error if you set the `re.MatchTimeout` field, or `ErrBacktrackingStackLimit` if a match exceeds its configured backtracking stack size. Any other error is a bug in the `regexp2` package. If you need more details about capture groups in a match then use the `FindStringMatch` method, like so: ```go if m, _ := re.FindStringMatch(`Something to match`); m != nil { @@ -92,6 +92,17 @@ notEmoji := regexp2.MustCompile(`\P{Emoji}+`) Valid property names and aliases come from Unicode 17.0.0 [`PropertyAliases.txt`](https://www.unicode.org/Public/17.0.0/ucd/PropertyAliases.txt). Valid property values and aliases come from Unicode 17.0.0 [`PropertyValueAliases.txt`](https://www.unicode.org/Public/17.0.0/ucd/PropertyValueAliases.txt). The generated tables use Unicode 17.0.0 data from [`DerivedCoreProperties.txt`](https://www.unicode.org/Public/17.0.0/ucd/DerivedCoreProperties.txt), [`emoji/emoji-data.txt`](https://www.unicode.org/Public/17.0.0/ucd/emoji/emoji-data.txt), [`auxiliary/GraphemeBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/GraphemeBreakProperty.txt), [`auxiliary/WordBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/WordBreakProperty.txt), and [`auxiliary/SentenceBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/SentenceBreakProperty.txt) for the package-local properties whose data changes more frequently than the Go standard library tables. +## Additional Perl and PCRE syntax + +The default mode supports the following syntax: + +* `\Q...\E` quotes every character between `\Q` and `\E`. If `\E` is omitted, quoting continues to the end of the pattern. This also works inside character classes. +* `\R` matches one Unicode newline sequence: CRLF as a single sequence, or LF, VT, FF, CR, NEL, line separator, or paragraph separator. +* `\X` atomically matches one Unicode 17.0.0 extended grapheme cluster. This includes combining sequences, Hangul syllables, regional-indicator pairs, emoji ZWJ sequences, and Indic conjuncts. +* A `+` after a quantifier makes it possessive: `*+`, `++`, `?+`, and `{m,n}+`. For example, `a*+` is equivalent to `(?>a*)` and will not give characters back when the remainder of the pattern fails. + +`RE2` mode also supports `\Q...\E` and possessive quantifiers, but keeps `\R` and `\X` as literal `R` and `X` identity escapes. `ECMAScript` mode does not enable any of this syntax: `\Q`, `\E`, `\R`, and `\X` retain ECMAScript identity-escape behavior, and possessive quantifiers remain invalid. + ## `regexp` compatibility adapter The `github.com/dlclark/regexp2/v2/compat` package provides an adapter for callers that want the same `Find*` and `Match*` method signatures as the standard library's `regexp.Regexp`, while still using the `regexp2` engine. @@ -128,7 +139,7 @@ func findWords(re compat.Matcher, input string) []string { } ``` -Because those standard-library method signatures do not return errors, the adapter panics if the wrapped regexp2 matcher returns an error such as a match timeout. Use the main `regexp2` APIs directly when you need to handle timeouts as errors. +Because those standard-library method signatures do not return errors, the adapter panics if the wrapped regexp2 matcher returns an error such as a match timeout or `ErrBacktrackingStackLimit`. Use the main `regexp2` APIs directly when you need to handle match errors directly. ## Compile options @@ -146,6 +157,7 @@ Performance tuning options override the default cache settings: ```go re := regexp2.MustCompile(`Your pattern`, regexp2.IgnoreCase, + regexp2.OptionMaxBacktrackingStackSize(200000), regexp2.OptionMaxCachedRuneBufferLength(64*1024), regexp2.OptionMaxCachedReplacerDataEntries(8), ) @@ -164,23 +176,30 @@ The defaults are intentionally bounded: | `OptionMaintainCaptureOrder()` | false | Parser capture-slot assignment for mixed named and unnamed captures. | None at match time. This changes compile-time capture numbering only. | Keeps named and unnamed captures in pattern order instead of appending named captures after unnamed captures. This can change numeric backreference meaning, so it is caller-controlled rather than an inline regex option. | | `OptionDebug()` | false | Compile dumps and runner tracing. | Debug output volume only. | Useful for diagnostics, but it can produce noisy output and slower traced matching. | | `OptionIsCodeGen()` | false | Compile-time find-optimization analysis for [`regexp2cg`](https://github.com/dlclark/regexp2cg). | Per compiled regexp, during `Compile` or `MustCompile`. | Enables more expensive analysis intended for generated engines. Do not use it for normal interpreter execution; the interpreter defaults intentionally avoid this extra compile-time cost. | +| `OptionMaxBacktrackingStackSize(n)` | 100,000 | The interpreter's per-match backtracking stack. | Per pooled runner. The initial allocation and subsequent growth are capped at the configured number of integer slots; the runner pool may retain stacks at their high-water size for reuse. | Lowering this bounds backtracking memory more tightly but may reject complex matches sooner with `ErrBacktrackingStackLimit`. Raising it permits deeper backtracking and increases possible memory use. A negative value disables the limit. | | `OptionMaxCachedRuneBufferLength(n)` | 256K runes | String APIs that run through pooled runners, such as `MatchString` and replacement-pattern `Replace`, when converting input strings to the engine's internal `[]rune` representation. | Process-wide shared `sync.Pool` retention by size class. This does not grow per compiled regexp or per input string; the practical working set follows recent and concurrent use across all regexps and can be dropped by GC. | Raising this lets calls use larger pooled rune buffers and can reduce allocations for repeated matches against large strings. Lowering it prevents larger buffers from being borrowed or returned, so large inputs allocate directly. | | `OptionMaxCachedReplaceBufferLength(n)` | 256 KB | Replacement-pattern `Replace` calls that build output through a shared byte buffer. | Process-wide shared `sync.Pool` retention by size class after replacement-pattern `Replace` runs. It does not grow from evaluator-based `ReplaceFunc` output and is shared across compiled regexps. | Raising this lets larger replacement outputs use pooled buffers and can reduce allocations. Lowering it prevents larger output buffers from being retained, so large replacements allocate directly. | | `OptionMaxCachedReplacerDataEntries(n)` | `16` | `Replace` with replacement pattern strings, after the replacement pattern is parsed into reusable replacement data. | Per compiled regexp. The cache grows as distinct cacheable replacement strings are used with `Replace`, up to this entry count. | Raising this helps when a single compiled regexp is used with many recurring replacement patterns. It increases per-regexp cache memory and lock-protected cache bookkeeping. Setting it to `0` disables this cache. | | `OptionMaxCachedReplacerDataBytes(n)` | 4 KB | The parsed replacement-pattern cache. Replacement strings longer than this are parsed for the call but not retained. | Per compiled regexp, combined with `OptionMaxCachedReplacerDataEntries`. Only replacement strings whose source text is at or below this size can add parsed data to the cache. | Raising this helps if large replacement patterns are reused. It can retain more memory per cached replacement. Lowering it avoids keeping unusual large replacement patterns around. | | `OptionDisableCharClassASCIIBitmap()` | false | Compile-time preparation of character classes and first-character prefix sets. By default, character classes with ASCII membership get a small bitmap used by `CharIn`. | Per compiled regexp, during `Compile` or `MustCompile`. Each eligible character class can hold one small bitmap; this does not scale with match concurrency or input size. | Leaving this false speeds up ASCII-heavy character class checks at the cost of a small amount of per-char-class memory and compile-time work. Setting to true can reduce memory for large numbers of compiled char classes in regexps, but ASCII character class matching may be slower. | +For `OptionMaxBacktrackingStackSize`, set `n` to a negative value to allow unbounded stack growth. Setting it to `0` permits no backtracking stack entries, so most interpreted matches will return `ErrBacktrackingStackLimit`. + For pooled buffer cache options, set `n` to `0` to disable pooling, or `-1` to allow all built-in size classes. The rune buffer classes are 1K, 4K, 16K, 64K, and 256K runes. The replacement byte buffer classes are 4 KB, 16 KB, 64 KB, 256 KB, and 1 MB. By default the 1 MB pool is unused. For replacement data byte-size cache options, `-1` means unbounded. For entry-count cache options, set `n` to `0` to disable the cache. ## Compare `regexp` and `regexp2` | Category | regexp | regexp2 | | --- | --- | --- | -| Catastrophic backtracking possible | no, constant execution time guarantees | yes, if your pattern is at risk you can use the `re.MatchTimeout` field | +| Catastrophic backtracking possible | no, constant execution time guarantees | yes; backtracking stack growth is bounded by default, and `re.MatchTimeout` can also bound match duration | | Python-style capture groups `(?Pre)` | yes | no (yes in RE2 compat mode) | | .NET-style capture groups `(?re)` or `(?'name're)` | yes | yes | | comments `(?#comment)` | no | yes | | branch numbering reset `(?\|a\|b)` | no | no | -| possessive match `(?>re)` | no | yes | +| atomic group `(?>re)` | no | yes | +| possessive quantifiers `*+`, `++`, `?+`, `{m,n}+` | no | yes | +| literal quoting `\Q...\E` | yes | yes | +| Unicode newline sequence `\R` | no | yes (default mode only) | +| extended grapheme cluster `\X` | no | yes (default mode only) | | positive lookahead `(?=re)` | no | yes | | negative lookahead `(?!re)` | no | yes | | positive lookbehind `(?<=re)` | no | yes | @@ -199,6 +218,8 @@ The default behavior of `regexp2` is to match the .NET regexp engine, however th * change singleline behavior for `$` to only match end of string (like RE2) (see [#24](https://github.com/dlclark/regexp2/issues/24)) * change the character classes `\d` `\s` and `\w` to match the same characters as RE2. NOTE: if you also use the `ECMAScript` option then this will change the `\s` character class to match ECMAScript instead of RE2. ECMAScript allows more whitespace characters in `\s` than RE2 (but still fewer than the the default behavior). * allow character escape sequences to have defaults. For example, by default `\_` isn't a known character escape and will fail to compile, but in RE2 mode it will match the literal character `_` +* support RE2-style literal quoting with `\Q...\E` +* support possessive quantifiers (`*+`, `++`, `?+`, and `{m,n}+`) as a regexp2 extension ```go re := regexp2.MustCompile(`Your RE2-compatible pattern`, regexp2.RE2) @@ -212,7 +233,22 @@ This feature is a work in progress and I'm open to ideas for more things to put ## Catastrophic Backtracking and Timeouts `regexp2` supports features that can lead to catastrophic backtracking. -`Regexp.MatchTimeout` can be set to to limit the impact of such behavior; the +Each compiled regexp limits its per-match backtracking stack to 100,000 +slots by default. If a match would exceed that limit, it stops and returns +`ErrBacktrackingStackLimit`. Callers can identify it with +`errors.Is(err, regexp2.ErrBacktrackingStackLimit)`. The limit can be changed at +compile time; a negative value restores the previous unbounded behavior: + +```go +re := regexp2.MustCompile(pattern, regexp2.OptionMaxBacktrackingStackSize(200000)) +// regexp2.OptionMaxBacktrackingStackSize(-1) disables the limit. +``` + +This limit bounds the interpreter's backtracking stack, not total match time or +all memory used by a match. Literal empty expressions repeated any number of +times are optimized away and do not consume backtracking stack space. + +`Regexp.MatchTimeout` can be set to limit the impact of such behavior; the match will fail with an error after approximately MatchTimeout. No timeout checks are done by default. @@ -276,6 +312,8 @@ This flag should not be treated as compatibility with C#'s `RegexOptions.ECMAScr Additionally a Unicode mode is provided which allows parsing of `\u{CodePoint}` syntax only when both `ECMAScript` and `Unicode` are provided. +Perl/PCRE extensions `\Q...\E`, `\R`, `\X`, and possessive quantifiers are intentionally not enabled in this mode. The letter escapes retain the engine's existing ECMAScript identity-escape behavior. + ## Potential bugs I've run a battery of tests against regexp2 from various sources and found the debug output matches the .NET engine, but .NET and Go handle strings very differently. I've attempted to handle these differences, but most of my testing deals with basic ASCII with a little bit of multi-byte Unicode. There's a chance that there are bugs in the string handling related to character sets with supplementary Unicode chars. Right-to-Left support is coded, but not well tested either. diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/decode.go b/backend/vendor/github.com/dlclark/regexp2/v2/decode.go new file mode 100644 index 00000000..6427e548 --- /dev/null +++ b/backend/vendor/github.com/dlclark/regexp2/v2/decode.go @@ -0,0 +1,152 @@ +package regexp2 + +import ( + "unicode/utf8" + + "github.com/dlclark/regexp2/v2/helpers" +) + +// decodedInput is the engine's rune view of a string. When the pattern cannot +// look behind a candidate start, the slice may begin at that candidate instead +// of byte 0. +type decodedInput struct { + runes []rune + pooled *[]rune + runeStart int // index in runes of the requested startAt; -1 if not a rune boundary + runeOffset int // original-string rune index of runes[0] + byteOffset int // original-string byte index of runes[0] +} + +// decodeFrom is the first original-string byte that must be decoded. 0 means +// the whole string; a later index is only used when slicing is safe. +func (re *Regexp) decodeFrom(s string, startAt int) int { + if startAt <= 0 || re.RightToLeft() { + return 0 + } + need := re.decodeLeftContextRunes() + if need < 0 { + return 0 + } + if need > 0 { + return prevRuneByte(s, startAt) + } + return startAt +} + +func (re *Regexp) decodeLeftContextRunes() int { + if re.code != nil { + return re.code.LeftContextRunes + } + return re.leftContextRunes +} + +// decodeString converts s to []rune for the MatchString startAt<=0 path. +// Keep this as close as possible to a single UTF-8 walk so 4–30 byte +// matches stay cheap. +func decodeString(s string, maxCachedLength int) ([]rune, *[]rune) { + buf, pooled := pooledRuneBuffers.get(len(s), maxCachedLength) + if len(s) >= helpers.ASCIIScanMin { + n, _ := helpers.DecodeString(s, buf) + return buf[:n], pooled + } + n := 0 + for _, ch := range s { + buf[n] = ch + n++ + } + return buf[:n], pooled +} + +func (re *Regexp) decodeStringInput(s string, startAt int, pooled bool) decodedInput { + maxCachedLength := 0 + if pooled { + maxCachedLength = re.optimizations.MaxCachedRuneBufferLength + } + return decodeInput(s, startAt, re.decodeFrom(s, startAt), maxCachedLength, true) +} + +// decodeInput converts s[decodeFrom:] to runes. startAt is a byte index in s. +// If startAt < 0 or is not a rune boundary, runeStart is -1. needOffsets walks +// the skipped prefix to fill runeOffset; MatchString does not need that field. +func decodeInput(s string, startAt, decodeFrom, maxCachedLength int, needOffsets bool) decodedInput { + if decodeFrom < 0 { + decodeFrom = 0 + } + if decodeFrom > len(s) { + decodeFrom = len(s) + } + + substr := s[decodeFrom:] + buf, pooledBuffer := pooledRuneBuffers.get(len(substr), maxCachedLength) + n, ascii := helpers.DecodeString(substr, buf) + runes := buf[:n] + + out := decodedInput{ + runes: runes, + pooled: pooledBuffer, + byteOffset: decodeFrom, + runeStart: startRuneIndex(s, startAt, decodeFrom, n, ascii), + } + + if !needOffsets { + return out + } + + // Byte index equals rune index only when the skipped prefix is also ASCII. + if ascii && (decodeFrom == 0 || helpers.IsASCII(s[:decodeFrom])) { + out.runeOffset = decodeFrom + return out + } + if decodeFrom > 0 { + out.runeOffset = utf8.RuneCountInString(s[:decodeFrom]) + } + return out +} + +func startRuneIndex(s string, startAt, decodeFrom, runeCount int, ascii bool) int { + if startAt < 0 { + return -1 + } + if startAt < decodeFrom || startAt > len(s) { + return -1 + } + if startAt == decodeFrom { + return 0 + } + if startAt == len(s) { + return runeCount + } + if ascii { + return startAt - decodeFrom + } + rel := startAt - decodeFrom + i := 0 + for strIdx := range s[decodeFrom:] { + if strIdx == rel { + return i + } + if strIdx > rel { + return -1 + } + i++ + } + return -1 +} + +func prevRuneByte(s string, byteIndex int) int { + if byteIndex <= 0 { + return 0 + } + _, size := utf8.DecodeLastRuneInString(s[:byteIndex]) + if size <= 0 { + return byteIndex - 1 + } + return byteIndex - size +} + +func (d decodedInput) release() { + if d.pooled != nil { + *d.pooled = d.runes + pooledRuneBuffers.put(d.pooled) + } +} diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/helpers/ascii.go b/backend/vendor/github.com/dlclark/regexp2/v2/helpers/ascii.go new file mode 100644 index 00000000..a7b4829a --- /dev/null +++ b/backend/vendor/github.com/dlclark/regexp2/v2/helpers/ascii.go @@ -0,0 +1,75 @@ +package helpers + +import ( + "unicode/utf8" + "unsafe" +) + +const asciiHighBits = 0x8080808080808080 + +// ASCIIScanMin is the input length where a separate ASCII probe plus a tight +// copy beats a single UTF-8 range loop. Short matches stay on the one-pass path. +const ASCIIScanMin = 64 + +// ASCIISearchMin is the haystack length where stdlib/SIMD string search beats +// a scalar first-rune scan. Below this, needle setup and unaligned +// bytes.Index retries dominate. +const ASCIISearchMin = 32 + +// IsASCII reports whether s contains only bytes below utf8.RuneSelf. +func IsASCII(s string) bool { + n := len(s) + if n == 0 { + return true + } + + b := unsafe.Slice(unsafe.StringData(s), n) + i := 0 + for ; i < n && uintptr(unsafe.Pointer(&b[i]))&7 != 0; i++ { + if b[i] >= utf8.RuneSelf { + return false + } + } + for ; i+8 <= n; i += 8 { + v := *(*uint64)(unsafe.Pointer(&b[i])) + if v&asciiHighBits != 0 { + return false + } + } + for ; i < n; i++ { + if b[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// IsASCIIRunes reports whether every rune in in is an ASCII code point. +func IsASCIIRunes(in []rune) bool { + for _, ch := range in { + if ch >= utf8.RuneSelf { + return false + } + } + return true +} + +// DecodeString writes the runes of s into buf, which must have length >= len(s). +// It returns the rune count and whether s was all ASCII. +func DecodeString(s string, buf []rune) (n int, ascii bool) { + if len(s) >= ASCIIScanMin && IsASCII(s) { + for i := 0; i < len(s); i++ { + buf[i] = rune(s[i]) + } + return len(s), true + } + ascii = true + for _, ch := range s { + if ch >= utf8.RuneSelf { + ascii = false + } + buf[n] = ch + n++ + } + return n, ascii +} diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/helpers/indexof.go b/backend/vendor/github.com/dlclark/regexp2/v2/helpers/indexof.go index 8d24a8c0..14dee3c2 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/helpers/indexof.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/helpers/indexof.go @@ -11,11 +11,30 @@ import ( ) func IndexOfAny(in []rune, find []rune) int { - // special case - if len(find) == 0 { + switch len(find) { + case 0: + return -1 + case 1: + return IndexOfAny1(in, find[0]) + case 2: + return IndexOfAny2(in, find[0], find[1]) + case 3: + return IndexOfAny3(in, find[0], find[1], find[2]) + } + + if IsASCIIRunes(find) { + var bits [2]uint64 + for _, c := range find { + bits[c>>6] |= 1 << (c & 63) + } + for i, c := range in { + if uint32(c) < 128 && bits[c>>6]&(1<<(c&63)) != 0 { + return i + } + } return -1 } - // naive version + for i, c := range in { if slices.Contains(find, c) { return i @@ -25,8 +44,10 @@ func IndexOfAny(in []rune, find []rune) int { } func IndexOfAny1(in []rune, find rune) int { - //TODO: bytes optimization? - return slices.Index(in, find) + if len(in) < ASCIISearchMin { + return slices.Index(in, find) + } + return indexOfRuneBytes(runeSliceBytes(in), find) } func IndexOfAny2(in []rune, find1, find2 rune) int { @@ -125,28 +146,34 @@ func IndexFunc(in []rune, f func(ch rune) bool) int { } func IndexOfAnyExceptInSet(in []rune, set syntax.CharSet) int { - //TODO: this - panic("not implemented") + for i, c := range in { + if !set.CharIn(c) { + return i + } + } + return -1 } func LastIndexOf(in []rune, find []rune) int { - end := len(in) - len(find) - first := find[0] - lastOffset := len(find) - 1 - last := find[lastOffset] - for i := end; i >= 0; i-- { - //TODO: check 2 chars needed? - // match start and end...check the middle - if in[i] == first && in[i+lastOffset] == last { - // found our first char - // check if the rest are equal - if bytesEqual(in[i:i+len(find)], find) { - return i - } + if len(find) == 0 { + return len(in) + } + if len(in) < len(find) { + return -1 + } + haystack := runeSliceBytes(in) + needle := runeSliceBytes(find) + end := len(haystack) + for end >= len(needle) { + idx := bytes.LastIndex(haystack[:end], needle) + if idx < 0 { + return -1 } + if idx%4 == 0 { + return idx / 4 + } + end = idx + len(needle) - 1 } - - //not found return -1 } @@ -160,15 +187,10 @@ func LastIndexOfAnyExcept1(in []rune, not rune) int { } func LastIndexOfAny1(in []rune, find rune) int { - for i := len(in) - 1; i >= 0; i-- { - if in[i] == find { - // found our char - return i - } + if len(in) == 0 { + return -1 } - - //not found - return -1 + return lastIndexOfRuneBytes(runeSliceBytes(in), find) } func LastIndexOfAnyInRange(in []rune, first, last rune) int { @@ -291,30 +313,66 @@ func foldASCII(c rune) rune { } func IndexOf(in []rune, find []rune) int { - /* - Since we auto-gen the find code this shouldn't happen - if len(find) == 0 { - //special case - return -1 - }*/ + if len(find) == 0 { + return 0 + } + if len(in) < len(find) { + return -1 + } end := len(in) - len(find) first := find[0] - //TODO: benchmark checking last char too or first two chars - for i := 0; i <= end; i++ { - // match start...check the rest - if in[i] == first { - // found our first char - // check if the rest are equal - if bytesEqual(in[i:i+len(find)], find) { - return i - } - /*if slices.Equal(in[i:i+len(find)], find) { - return i - }*/ + for i := 0; i <= end; { + off := IndexOfAny1(in[i:end+1], first) + if off < 0 { + return -1 } + i += off + if bytesEqual(in[i:i+len(find)], find) { + return i + } + i++ } + return -1 +} - //not found +func runeSliceBytes(in []rune) []byte { + if len(in) == 0 { + return nil + } + return unsafe.Slice((*byte)(unsafe.Pointer(&in[0])), len(in)*4) +} + +func indexOfRuneBytes(haystack []byte, find rune) int { + needleRune := [1]rune{find} + needle := runeSliceBytes(needleRune[:]) + start := 0 + for { + idx := bytes.Index(haystack[start:], needle) + if idx < 0 { + return -1 + } + idx += start + if idx%4 == 0 { + return idx / 4 + } + start = idx + 1 + } +} + +func lastIndexOfRuneBytes(haystack []byte, find rune) int { + needleRune := [1]rune{find} + needle := runeSliceBytes(needleRune[:]) + end := len(haystack) + for end >= 4 { + idx := bytes.LastIndex(haystack[:end], needle) + if idx < 0 { + return -1 + } + if idx%4 == 0 { + return idx / 4 + } + end = idx + 3 + } return -1 } diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/helpers/searchvalues.go b/backend/vendor/github.com/dlclark/regexp2/v2/helpers/searchvalues.go index bafbcfbc..0346d7d4 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/helpers/searchvalues.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/helpers/searchvalues.go @@ -64,14 +64,30 @@ func (s AsciiSearchValues) IndexOfAnyExcept(chars []rune) int { // return the last index of our original vals values within the slice given func (s AsciiSearchValues) LastIndexOfAny(chars []rune) int { - panic("not implemented") - //TODO: this + for i := len(chars) - 1; i >= 0; i-- { + c := chars[i] + if c > unicode.MaxASCII { + continue + } + if s.set[c/64]&(1<<(c%64)) != 0 { + return i + } + } + return -1 } // return the last index of our original vals values within the slice given func (s AsciiSearchValues) LastIndexOfAnyExcept(chars []rune) int { - panic("not implemented") - //TODO: this + for i := len(chars) - 1; i >= 0; i-- { + c := chars[i] + if c > unicode.MaxASCII { + return i + } + if s.set[c/64]&(1<<(c%64)) == 0 { + return i + } + } + return -1 } type RuneSearchValues struct { @@ -79,37 +95,40 @@ type RuneSearchValues struct { } func newRuneSearchValues(vals []rune) RuneSearchValues { - //TODO: pre-calc the stuff we need to make each IndexOf go faster return RuneSearchValues{vals: vals} - } func NewRuneSearchValues(vals string) RuneSearchValues { return newRuneSearchValues([]rune(vals)) } -// return the first index of our original vals values within the slice given func (s RuneSearchValues) IndexOfAny(chars []rune) int { - //naive implementation - //TODO: this return IndexOfAny(chars, s.vals) } -// return the first index of our original vals values within the slice given func (s RuneSearchValues) IndexOfAnyExcept(chars []rune) int { - //TODO: this return IndexOfAnyExcept(chars, s.vals) } -// return the last index of our original vals values within the slice given func (s RuneSearchValues) LastIndexOfAny(chars []rune) int { - panic("not implemented") + if len(s.vals) == 1 { + return LastIndexOfAny1(chars, s.vals[0]) + } + for i := len(chars) - 1; i >= 0; i-- { + if slices.Contains(s.vals, chars[i]) { + return i + } + } + return -1 } -// return the last index of our original vals values within the slice given func (s RuneSearchValues) LastIndexOfAnyExcept(chars []rune) int { - panic("not implemented") - //TODO: this + for i := len(chars) - 1; i >= 0; i-- { + if !slices.Contains(s.vals, chars[i]) { + return i + } + } + return -1 } type StringSearchValues struct { @@ -145,11 +164,21 @@ func NewStringSearchValues(vals [][]rune, ignoreCase bool) StringSearchValues { } func (s StringSearchValues) StartsWith(chars []rune) int { - panic("not implemented") + for _, val := range s.vals { + if StartsWith(chars, val) { + return 0 + } + } + return -1 } func (s StringSearchValues) StartsWithIgnoreCase(chars []rune) int { - panic("not implemented") + for _, val := range s.vals { + if StartsWithIgnoreCase(chars, val) { + return 0 + } + } + return -1 } func (s StringSearchValues) IndexOfAny(in []rune) int { diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/match.go b/backend/vendor/github.com/dlclark/regexp2/v2/match.go index 4003ff0f..ba8a0105 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/match.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/match.go @@ -44,8 +44,9 @@ type Group struct { type Capture struct { // the original string text *matchText - // RuneIndex is the position in the underlying rune slice where the first character of - // captured substring was found. Even if you pass in a string this will be in Runes. + // RuneIndex is the rune index in the original input where the capture starts. + // For string input this counts runes from the start of that string, not of + // any internally sliced decode buffer. RuneIndex int // RuneLength is the number of runes in the captured substring. RuneLength int @@ -55,24 +56,46 @@ type matchText struct { runes []rune input string hasStringInput bool + runeOffset int // original-string rune index of runes[0] + byteOffset int // original-string byte index of runes[0] byteOffsets []int byteOffsetsReady bool } -// String returns the captured text as a String +// String returns the captured text. For string input it is a slice of the +// original haystack: it does not allocate and keeps the original string alive. func (c *Capture) String() string { - return string(c.text.runes[c.RuneIndex : c.RuneIndex+c.RuneLength]) + if c.text == nil { + return "" + } + if c.text.hasStringInput { + start, length := c.ByteRange() + return c.text.input[start : start+length] + } + start := c.runeSliceIndex() + return string(c.text.runes[start : start+c.RuneLength]) } // Runes returns the captured text as a rune slice func (c *Capture) Runes() []rune { - return c.text.runes[c.RuneIndex : c.RuneIndex+c.RuneLength] + if c.text == nil { + return nil + } + start := c.runeSliceIndex() + return c.text.runes[start : start+c.RuneLength] +} + +func (c *Capture) runeSliceIndex() int { + if c.text == nil { + return c.RuneIndex + } + return c.RuneIndex - c.text.runeOffset } // ByteRange returns the UTF-8 byte index and byte length of the captured -// substring. The first call lazily caches byte offsets on shared match text, -// so it is not safe to call concurrently with ByteRange on another capture -// from the same match until the cache has been initialized. +// substring. Matches returned to callers have offsets computed when the match +// is tidied, so concurrent ByteRange/String on captures of the same match is +// then safe. Internal match objects may still initialize the cache on first use. func (c *Capture) ByteRange() (index, length int) { if c.text == nil { return c.RuneIndex, c.RuneLength @@ -85,24 +108,40 @@ func newMatchText(r []rune) *matchText { } func newStringMatchText(input string, r []rune) *matchText { - return &matchText{runes: r, input: input, hasStringInput: true} + return newStringMatchTextAt(input, r, 0, 0) +} + +func newStringMatchTextAt(input string, r []rune, runeOffset, byteOffset int) *matchText { + return &matchText{ + runes: r, + input: input, + hasStringInput: true, + runeOffset: runeOffset, + byteOffset: byteOffset, + } +} + +func (t *matchText) ensureByteOffsets() { + if t == nil || t.byteOffsetsReady { + return + } + t.byteOffsets = t.buildByteOffsets() + t.byteOffsetsReady = true } func (t *matchText) byteRange(runeIndex, runeLength int) (int, int) { - if !t.byteOffsetsReady { - t.byteOffsets = t.buildByteOffsets() - t.byteOffsetsReady = true - } + localRuneIndex := runeIndex - t.runeOffset + t.ensureByteOffsets() if t.byteOffsets == nil { - return runeIndex, runeLength + return t.byteOffset + localRuneIndex, runeLength } - byteIndex := t.byteOffsets[runeIndex] - return byteIndex, t.byteOffsets[runeIndex+runeLength] - byteIndex + byteIndex := t.byteOffsets[localRuneIndex] + return t.byteOffset + byteIndex, t.byteOffsets[localRuneIndex+runeLength] - byteIndex } func (t *matchText) buildByteOffsets() []int { if t.hasStringInput { - return stringByteOffsets(t.input) + return stringByteOffsets(t.input[t.byteOffset:]) } return runeByteOffsets(t.runes) } @@ -199,6 +238,9 @@ func (m *Match) tidy(textpos int) { m.capcount = m.matchcount[0] //copy our root capture to the list m.Captures = []Capture{m.Capture} + if m.text != nil && m.text.hasStringInput { + m.text.ensureByteOffsets() + } if m.balancing { // The idea here is that we want to compact all of our unbalanced captures. To do that we @@ -415,6 +457,9 @@ func newCapture(text *matchText, runeIndex, runeLength int) Capture { } func setCaptureFields(c *Capture, runeIndex, runeLength int) { + if c.text != nil { + runeIndex += c.text.runeOffset + } c.RuneIndex = runeIndex c.RuneLength = runeLength } diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/options.go b/backend/vendor/github.com/dlclark/regexp2/v2/options.go index c0e49fff..e54fa782 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/options.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/options.go @@ -5,6 +5,7 @@ var ( DefaultUnmarshalOptions = None // DefaultOptimizationOptions controls the default memory/performance trade-offs used by Compile. DefaultOptimizationOptions = OptimizationOptions{ + MaxBacktrackingStackSize: 100000, MaxCachedRuneBufferLength: 256 << 10, MaxCachedReplaceBufferLength: 256 << 10, MaxCachedReplacerDataEntries: 16, @@ -36,13 +37,17 @@ const ( Unicode RegexOptions = 0x0400 // "u" ) -// OptimizationOptions controls optional runtime caches and compile-time fast paths. +// OptimizationOptions controls runtime limits, optional caches, and compile-time fast paths. // +// For MaxBacktrackingStackSize, negative values allow unbounded growth. // For replacement data cache size fields, 0 disables persistent retention and // -1 means unbounded. For pooled buffer cache size fields, 0 disables pooling // and -1 allows all built-in size classes. // Defaults are intentionally bounded so Compile is safe for mixed-cardinality inputs. type OptimizationOptions struct { + // MaxBacktrackingStackSize limits the number of integer slots used by a match's backtracking stack. + // Negative values disable the limit. + MaxBacktrackingStackSize int // MaxCachedRuneBufferLength limits retained string-to-rune buffers in the shared size-classed pool. MaxCachedRuneBufferLength int // MaxCachedReplaceBufferLength limits retained replacement output buffers in the shared size-classed pool. @@ -100,6 +105,14 @@ func newCompileConfig(options []CompileOption) compileConfig { return c } +// OptionMaxBacktrackingStackSize limits the number of integer slots used by a match's backtracking stack. +// Negative values disable the limit. A match that exceeds the limit returns ErrBacktrackingStackLimit. +func OptionMaxBacktrackingStackSize(n int) CompileOption { + return compileOptionFunc(func(c *compileConfig) { + c.optimizations.MaxBacktrackingStackSize = n + }) +} + // OptionMaxCachedRuneBufferLength limits retained string-to-rune buffers in the shared size-classed pool. func OptionMaxCachedRuneBufferLength(n int) CompileOption { return compileOptionFunc(func(c *compileConfig) { diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/regexp.go b/backend/vendor/github.com/dlclark/regexp2/v2/regexp.go index 0a5d140a..c544a572 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/regexp.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/regexp.go @@ -25,6 +25,8 @@ import ( var ( // DefaultMatchTimeout used when running regexp matches -- "forever" DefaultMatchTimeout = time.Duration(math.MaxInt64) + // ErrBacktrackingStackLimit is returned when a match exceeds its configured backtracking stack size. + ErrBacktrackingStackLimit = errors.New("regexp2: maximum backtracking stack size exceeded") ) // Regexp is the representation of a compiled regular expression. @@ -59,7 +61,12 @@ type Regexp struct { // hook points to override runner functions findFirstChar func(r *Runner) bool execute func(r *Runner) error + executeQuick func(r *Runner) error stringPrefixFilter StringPrefixFilter + quickCode *syntax.Code // bool-only program with unobservable captures removed + // leftContextRunes is used when code is nil (registered engines). + // The interpreter reads the same value from code.LeftContextRunes. + leftContextRunes int } // Compile parses a regular expression and returns, if successful, @@ -107,6 +114,7 @@ func compile(expr string, c compileConfig) (*Regexp, error) { capslist: tree.Caplist, capsize: code.Capsize, code: code, + quickCode: makeQuickCode(code), MatchTimeout: DefaultMatchTimeout, optimizations: c.optimizations, } @@ -115,6 +123,18 @@ func compile(expr string, c compileConfig) (*Regexp, error) { return re, nil } +func makeQuickCode(code *syntax.Code) *syntax.Code { + if code == nil || len(code.QuickCodes) == 0 { + return nil + } + quick := *code + quick.Codes = code.QuickCodes + quick.Dispatches = code.QuickDispatches + quick.QuickCodes = nil + quick.QuickDispatches = nil + return &quick +} + // MustCompile is like Compile but panics if the expression cannot be parsed. // It simplifies safe initialization of global variables holding compiled regular // expressions. @@ -225,17 +245,12 @@ func (re *Regexp) FindStringMatch(s string) (*Match, error) { if !ok { return nil, nil } - - r, runeStart := re.getRunesAndStart(s, startAt) - if runeStart < 0 { - runeStart = 0 - } - return re.run(false, runeStart, r, newStringMatchText(s, r)) + return re.findDecodedStringMatch(s, startAt) } // FindRunesMatch searches the input rune slice for a Regexp match func (re *Regexp) FindRunesMatch(r []rune) (*Match, error) { - return re.run(false, -1, r, newMatchText(r)) + return re.run(false, -1, -1, r, newMatchText(r)) } // FindStringMatchStartingAt searches the input string for a Regexp match starting at the startAt index @@ -247,19 +262,22 @@ func (re *Regexp) FindStringMatchStartingAt(s string, startAt int) (*Match, erro if !ok { return nil, nil } + return re.findDecodedStringMatch(s, startAt) +} - r, startAt := re.getRunesAndStart(s, startAt) - if startAt == -1 { - // we didn't find our start index in the string -- that's a problem - return nil, errors.New("startAt must align to the start of a valid rune in the input string") - } - - return re.run(false, startAt, r, newStringMatchText(s, r)) +func (re *Regexp) findDecodedStringMatch(s string, startAt int) (*Match, error) { + // Returned matches retain their rune data, so this path must not consume a + // pooled buffer that can never be returned. + d := re.decodeStringInput(s, startAt, false) + runner := re.getRunner() + defer re.putRunner(runner) + text := newStringMatchTextAt(s, d.runes, d.runeOffset, d.byteOffset) + return runner.scan(d.runes, text, d.runeStart, -1, false, re.MatchTimeout) } // FindRunesMatchStartingAt searches the input rune slice for a Regexp match starting at the startAt index func (re *Regexp) FindRunesMatchStartingAt(r []rune, startAt int) (*Match, error) { - return re.run(false, startAt, r, newMatchText(r)) + return re.run(false, startAt, -1, r, newMatchText(r)) } // FindAllStringIndex returns a slice of byte index pairs identifying all @@ -277,33 +295,22 @@ func (re *Regexp) FindAllStringIndex(s string, n int) ([][]int, error) { return nil, nil } + d := re.decodeStringInput(s, startAt, true) runner := re.getRunner() - var input []rune - var pooledInput *[]rune - runeStart := 0 - if startAt == 0 { - input, pooledInput = runner.decodeString(s) - } else { - input, runeStart, pooledInput = runner.decodeStringWithStart(s, startAt) - } defer func() { re.putRunner(runner) - if pooledInput != nil { - *pooledInput = input - pooledRuneBuffers.put(pooledInput) - } + d.release() }() - - if runeStart < 0 { - runeStart = 0 - } - byteOffsets := newStringByteMapper(s) - return re.findAllRunesIndex(runner, input, runeStart, n, func(runeIndex, runeLength int) (int, int) { + if re.quickCode != nil { + runner.code = re.quickCode + } + return re.findAllRunesIndex(runner, d.runes, d.runeStart, n, func(runeIndex, runeLength int) (int, int) { if byteOffsets == nil { - return runeIndex, runeIndex + runeLength + return d.byteOffset + runeIndex, d.byteOffset + runeIndex + runeLength } - return byteOffsets.byteIndex(runeIndex), byteOffsets.byteIndex(runeIndex + runeLength) + start := runeIndex + d.runeOffset + return byteOffsets.byteIndex(start), byteOffsets.byteIndex(start + runeLength) }) } @@ -321,6 +328,9 @@ func (re *Regexp) FindAllRunesIndex(r []rune, n int) ([][]int, error) { if re.RightToLeft() { startAt = len(r) } + if re.quickCode != nil { + runner.code = re.quickCode + } return re.findAllRunesIndex(runner, r, startAt, n, func(runeIndex, runeLength int) (int, int) { return runeIndex, runeIndex + runeLength }) @@ -335,8 +345,9 @@ func (re *Regexp) findAllRunesIndex(runner *Runner, input []rune, startAt, n int } prevEnd := -1 + previousMatchLength := -1 for n != 0 { - m, err := runner.scan(input, nil, startAt, true, re.MatchTimeout) + m, err := runner.scan(input, nil, startAt, previousMatchLength, true, re.MatchTimeout) if err != nil { return nil, err } @@ -344,34 +355,19 @@ func (re *Regexp) findAllRunesIndex(runner *Runner, input []rune, startAt, n int break } - if m.RuneLength != 0 || m.RuneIndex != prevEnd { - start, end := makeIndex(m.RuneIndex, m.RuneLength) + localIndex := m.runeSliceIndex() + if m.RuneLength != 0 || localIndex != prevEnd { + start, end := makeIndex(localIndex, m.RuneLength) flat = append(flat, start, end) out = append(out, flat[len(flat)-2:len(flat):len(flat)]) - prevEnd = m.RuneIndex + m.RuneLength + prevEnd = localIndex + m.RuneLength if n > 0 { n-- } } startAt = m.textpos - if m.RuneLength == 0 { - if re.RightToLeft() { - if m.textpos == 0 { - break - } - if startAt == m.textstart { - startAt-- - } - } else { - if m.textpos == len(input) { - break - } - if startAt == m.textstart { - startAt++ - } - } - } + previousMatchLength = m.RuneLength } return out, nil } @@ -420,29 +416,7 @@ func (re *Regexp) FindNextMatch(m *Match) (*Match, error) { return nil, nil } - // If previous match was empty, advance by one before matching to prevent - // infinite loop - startAt := m.textpos - if m.RuneLength == 0 { - if re.RightToLeft() { - if m.textpos == 0 { - return nil, nil - } - if startAt == m.textstart { - startAt-- - } - } else { - if m.textpos == len(m.text.runes) { - return nil, nil - } - - if startAt == m.textstart { - startAt++ - } - - } - } - return re.run(false, startAt, m.text.runes, m.text) + return re.run(false, m.textpos, m.RuneLength, m.text.runes, m.text) } // MatchString return true if the string matches the regex @@ -469,12 +443,16 @@ func (re *Regexp) matchStringAt(s string, startAt int) (bool, error) { var pooledInput *[]rune runeStart := 0 if startAt <= 0 { - input, pooledInput = runner.decodeString(s) + // Common path: decode the whole string without start/offset work. + input, pooledInput = decodeString(s, re.optimizations.MaxCachedRuneBufferLength) if re.RightToLeft() { runeStart = len(input) } } else { - input, runeStart, pooledInput = runner.decodeStringWithStart(s, startAt) + d := decodeInput(s, startAt, re.decodeFrom(s, startAt), re.optimizations.MaxCachedRuneBufferLength, false) + input = d.runes + pooledInput = d.pooled + runeStart = d.runeStart if runeStart < 0 { runeStart = 0 } @@ -486,46 +464,21 @@ func (re *Regexp) matchStringAt(s string, startAt int) (bool, error) { pooledRuneBuffers.put(pooledInput) } }() + if re.quickCode != nil { + runner.code = re.quickCode + } - m, err := runner.scan(input, nil, runeStart, true, re.MatchTimeout) + m, err := runner.scan(input, nil, runeStart, -1, true, re.MatchTimeout) if err != nil { return false, err } return m != nil, nil } -func (re *Regexp) getRunesAndStart(s string, startAt int) ([]rune, int) { - if startAt < 0 { - if re.RightToLeft() { - r := getRunes(s) - return r, len(r) - } - return getRunes(s), 0 - } - ret := make([]rune, len(s)) - i := 0 - runeIdx := -1 - for strIdx, r := range s { - if strIdx == startAt { - runeIdx = i - } - ret[i] = r - i++ - } - if startAt == len(s) { - runeIdx = i - } - return ret[:i], runeIdx -} - -func getRunes(s string) []rune { - return []rune(s) -} - // MatchRunes return true if the runes matches the regex // error will be set if a timeout occurs func (re *Regexp) MatchRunes(r []rune) (bool, error) { - m, err := re.run(true, -1, r, nil) + m, err := re.run(true, -1, -1, r, nil) if err != nil { return false, err } diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/regexp_codegen.go b/backend/vendor/github.com/dlclark/regexp2/v2/regexp_codegen.go index 41c1aef3..52248772 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/regexp_codegen.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/regexp_codegen.go @@ -11,13 +11,23 @@ type RuntimeEngineData struct { CapSize int // size of the capture array FindFirstChar func(*Runner) bool // generated candidate search Execute func(*Runner) error - StringPrefixFilter StringPrefixFilter // optional pre-decode candidate search for string input + ExecuteQuick func(*Runner) error // optional bool-only execution with unobservable captures removed + StringPrefixFilter StringPrefixFilter // optional pre-decode candidate search for string input + // LeftContextKnown reports that LeftContextRunes was computed by the + // code generator. If it is false, decoded string input is never sliced; + // older generated engines omit the field and must keep the full string. + LeftContextKnown bool + // LeftContextRunes is how many runes before a candidate start matching + // may inspect. 0 means none, 1 means a single previous rune, and -1 + // means do not slice (lookbehind or \G). Ignored unless LeftContextKnown. + LeftContextRunes int } type cacheKey struct { - pattern string - opt RegexOptions - maintainCaptureOrder bool + pattern string + opt RegexOptions + maintainCaptureOrder bool + maxBacktrackingStackSize int } func RegisterEngine(pattern string, engine RuntimeEngineData, options ...CompileOption) { @@ -28,6 +38,10 @@ func RegisterEngine(pattern string, engine RuntimeEngineData, options ...Compile } func newEngineRegexp(pattern string, c compileConfig, engine RuntimeEngineData) *Regexp { + leftContext := -1 + if engine.LeftContextKnown { + leftContext = engine.LeftContextRunes + } re := &Regexp{ pattern: pattern, options: c.regexOptions, @@ -40,7 +54,9 @@ func newEngineRegexp(pattern string, c compileConfig, engine RuntimeEngineData) optimizations: c.optimizations, findFirstChar: engine.FindFirstChar, execute: engine.Execute, + executeQuick: engine.ExecuteQuick, stringPrefixFilter: engine.StringPrefixFilter, + leftContextRunes: leftContext, } re.initCaches() return re @@ -58,9 +74,10 @@ func getEngineRegexp(pattern string, c compileConfig) *Regexp { func cacheKeyFromConfig(pattern string, c compileConfig) cacheKey { return cacheKey{ - pattern: pattern, - opt: c.regexOptions, - maintainCaptureOrder: c.maintainCaptureOrder, + pattern: pattern, + opt: c.regexOptions, + maintainCaptureOrder: c.maintainCaptureOrder, + maxBacktrackingStackSize: c.optimizations.MaxBacktrackingStackSize, } } diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/replace.go b/backend/vendor/github.com/dlclark/regexp2/v2/replace.go index 240b40f4..48739ab7 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/replace.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/replace.go @@ -87,15 +87,15 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator, } buf := &bytes.Buffer{} - text := m.text.runes if !regex.RightToLeft() { prevat := 0 for m != nil { - if m.RuneIndex != prevat { - buf.WriteString(string(text[prevat:m.RuneIndex])) + start, end := matchInputSpan(m) + if start > prevat { + buf.WriteString(input[prevat:start]) } - prevat = m.RuneIndex + m.RuneLength + prevat = end buf.WriteString(evaluator(*m)) count-- @@ -108,18 +108,19 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator, } } - if prevat < len(text) { - buf.WriteString(string(text[prevat:])) + if prevat < len(input) { + buf.WriteString(input[prevat:]) } } else { - prevat := len(text) + prevat := len(input) var al []string for m != nil { - if m.RuneIndex+m.RuneLength != prevat { - al = append(al, string(text[m.RuneIndex+m.RuneLength:prevat])) + start, end := matchInputSpan(m) + if end < prevat { + al = append(al, input[end:prevat]) } - prevat = m.RuneIndex + prevat = start al = append(al, evaluator(*m)) count-- @@ -133,7 +134,7 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator, } if prevat > 0 { - buf.WriteString(string(text[:prevat])) + buf.WriteString(input[:prevat]) } for i := len(al) - 1; i >= 0; i-- { @@ -144,20 +145,29 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator, return buf.String(), nil } +// matchInputSpan returns the UTF-8 byte range of m in its original string input. +func matchInputSpan(m *Match) (start, end int) { + if m.text != nil && m.text.hasStringInput { + start, length := m.ByteRange() + return start, start + length + } + return m.RuneIndex, m.RuneIndex + m.RuneLength +} + func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, startAt, count int) (string, error) { if startAt > len(input) { return "", errors.New("startAt must be less than the length of the input string") } runner := regex.getRunner() - text, runeStart, pooledText := runner.decodeStringWithStart(input, startAt) + d := decodeInput(input, startAt, 0, regex.optimizations.MaxCachedRuneBufferLength, false) + text := d.runes textInfo := newStringMatchText(input, text) defer func() { regex.putRunner(runner) - if pooledText != nil { - pooledRuneBuffers.put(pooledText) - } + d.release() }() + runeStart := d.runeStart if startAt >= 0 && runeStart < 0 { return "", errors.New("startAt must align to the start of a valid rune in the input string") } @@ -165,7 +175,7 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st runeStart = 0 } - m, err := runner.scan(text, textInfo, runeStart, true, regex.MatchTimeout) + m, err := runner.scan(text, textInfo, runeStart, -1, true, regex.MatchTimeout) if err != nil { return "", err } @@ -184,10 +194,11 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st compactBalancedMatches(m) } - if m.RuneIndex != prevat { - writeRunes(buf, text, prevat, m.RuneIndex) + local := m.runeSliceIndex() + if local != prevat { + writeRunes(buf, text, prevat, local) } - prevat = m.RuneIndex + m.RuneLength + prevat = local + m.RuneLength replacementImpl(data, buf, m) count-- @@ -195,15 +206,7 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st break } - scanStart := m.textpos - if m.RuneLength == 0 { - if scanStart >= len(text) { - break - } - scanStart++ - } - - m, err = runner.scan(text, textInfo, scanStart, true, regex.MatchTimeout) + m, err = runner.scan(text, textInfo, m.textpos, m.RuneLength, true, regex.MatchTimeout) if err != nil { return "", err } @@ -221,14 +224,14 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st } runner := regex.getRunner() - text, runeStart, pooledText := runner.decodeStringWithStart(input, startAt) + d := decodeInput(input, startAt, 0, regex.optimizations.MaxCachedRuneBufferLength, false) + text := d.runes textInfo := newStringMatchText(input, text) defer func() { regex.putRunner(runner) - if pooledText != nil { - pooledRuneBuffers.put(pooledText) - } + d.release() }() + runeStart := d.runeStart if startAt >= 0 && runeStart < 0 { return "", errors.New("startAt must align to the start of a valid rune in the input string") } @@ -236,7 +239,7 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st runeStart = len(text) } - m, err := runner.scan(text, textInfo, runeStart, true, regex.MatchTimeout) + m, err := runner.scan(text, textInfo, runeStart, -1, true, regex.MatchTimeout) if err != nil { return "", err } @@ -257,10 +260,11 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st compactBalancedMatches(m) } - if m.RuneIndex+m.RuneLength != prevat { - al = append(al, string(text[m.RuneIndex+m.RuneLength:prevat])) + local := m.runeSliceIndex() + if local+m.RuneLength != prevat { + al = append(al, string(text[local+m.RuneLength:prevat])) } - prevat = m.RuneIndex + prevat = local replacementImplRTL(data, &al, m) count-- @@ -268,15 +272,7 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st break } - scanStart := m.textpos - if m.RuneLength == 0 { - if scanStart <= 0 { - break - } - scanStart-- - } - - m, err = runner.scan(text, textInfo, scanStart, true, regex.MatchTimeout) + m, err = runner.scan(text, textInfo, m.textpos, m.RuneLength, true, regex.MatchTimeout) if err != nil { return "", err } @@ -303,11 +299,12 @@ func replacementImpl(data *syntax.ReplacerData, buf *bytes.Buffer, m *Match) { } else { switch -replaceSpecials - 1 - r { // special insertion patterns case replaceLeftPortion: - for i := 0; i < m.RuneIndex; i++ { + end := m.runeSliceIndex() + for i := 0; i < end; i++ { buf.WriteRune(m.text.runes[i]) } case replaceRightPortion: - for i := m.RuneIndex + m.RuneLength; i < len(m.text.runes); i++ { + for i := m.runeSliceIndex() + m.RuneLength; i < len(m.text.runes); i++ { buf.WriteRune(m.text.runes[i]) } case replaceLastGroup: @@ -335,11 +332,12 @@ func replacementImplRTL(data *syntax.ReplacerData, al *[]string, m *Match) { } else { switch -replaceSpecials - 1 - r { // special insertion patterns case replaceLeftPortion: - for i := 0; i < m.RuneIndex; i++ { + end := m.runeSliceIndex() + for i := 0; i < end; i++ { buf.WriteRune(m.text.runes[i]) } case replaceRightPortion: - for i := m.RuneIndex + m.RuneLength; i < len(m.text.runes); i++ { + for i := m.runeSliceIndex() + m.RuneLength; i < len(m.text.runes); i++ { buf.WriteRune(m.text.runes[i]) } case replaceLastGroup: diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/runner.go b/backend/vendor/github.com/dlclark/regexp2/v2/runner.go index ce09df07..6362cebf 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/runner.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/runner.go @@ -74,8 +74,10 @@ type Runner struct { // // quick is usually false, but can be true to not return matches, just put it in caches. // textstart is -1 to start at the "beginning" (depending on Right-To-Left), otherwise an index in input. +// previousMatchLength is -1 for an initial scan. A zero value advances the current scan position while +// preserving textstart for anchors such as \G. // textInfo is nil for quick scans that do not need returned capture text metadata. -func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchText) (*Match, error) { +func (re *Regexp) run(quick bool, textstart, previousMatchLength int, input []rune, textInfo *matchText) (*Match, error) { // get a cached runner runner := re.getRunner() @@ -88,8 +90,11 @@ func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchTe textstart = 0 } } + if quick && textInfo == nil && re.quickCode != nil { + runner.code = re.quickCode + } - return runner.scan(input, textInfo, textstart, quick, re.MatchTimeout) + return runner.scan(input, textInfo, textstart, previousMatchLength, quick, re.MatchTimeout) } // Scans the string to find the first match. Uses the Match object @@ -108,13 +113,16 @@ func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchTe // used as a boolean result and capture text is intentionally unavailable. If // we collapsed down to just textInfo it would "escape" and hit the GC for fast // scans without captures. -func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool, timeout time.Duration) (*Match, error) { +func (r *Runner) scan(rt []rune, textInfo *matchText, textstart, previousMatchLength int, quick bool, timeout time.Duration) (*Match, error) { r.timeout = timeout r.ignoreTimeout = (time.Duration(math.MaxInt64) == timeout) r.debug = r.re.Debug() r.Runtextstart = textstart r.Runtext = rt r.Runtextend = len(rt) + // Some internal callers use quick match tidying while still consuming + // capture data (notably replacement). Capture elision is only safe when no + // match text metadata was requested. stoppos := r.Runtextend bump := 1 @@ -130,6 +138,9 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool, // setup our scanner functions findFirstChar := r.re.findFirstChar execute := r.re.execute + if quick && textInfo == nil && r.re.executeQuick != nil { + execute = r.re.executeQuick + } if findFirstChar == nil { findFirstChar = findFirstCharDefault } @@ -144,6 +155,16 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool, r.initMatch(textInfo) + // An empty previous match must not be returned again. Keep Runtextstart at + // the previous match position for \G, but move the candidate scan position. + if previousMatchLength == 0 { + if r.Runtextpos == stoppos { + r.tidyMatch(true) + return nil, nil + } + r.Runtextpos += bump + } + r.startTimeoutWatch() for { if minRequiredLength > 0 { @@ -208,8 +229,9 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool, func executeDefault(r *Runner) error { - r.goTo(0) - + if err := r.goTo(0); err != nil { + return err + } for { if r.debug { @@ -230,7 +252,59 @@ func executeDefault(r *Runner) error { //noop case syntax.Goto: - r.goTo(r.operand(0)) + if err := r.goTo(r.operand(0)); err != nil { + return err + } + continue + + case syntax.Dispatch: + // Dispatch only peeks at the next rune. It consumes it after finding a + // matching branch, so a failed dispatch leaves the input position alone. + if r.forwardchars() < 1 { + break + } + // Pick the next rune in the current execution direction. + pos := r.Runtextpos + if r.rightToLeft { + pos-- + } + ch := r.Runtext[pos] + tableIndex := r.operand(0) + table := &r.code.Dispatches[tableIndex] + branch := -1 + if ch >= 0 && ch < 128 { + if table.ASCII != nil { + // Larger dispatches use a direct ASCII branch lookup. + branch = int(table.ASCII[ch]) - 1 + } else { + // Smaller dispatches use two compact ASCII bitmasks per set. + word := int(ch >> 6) + bit := uint64(1) << (ch & 63) + for i := range table.Sets { + if table.ASCIIMasks[i*2+word]&bit != 0 { + branch = i + break + } + } + } + } else { + // Non-ASCII runes fall back to the complete character sets. + for i, setIndex := range table.Sets { + if r.code.Sets[setIndex].CharIn(ch) { + branch = i + break + } + } + } + if branch < 0 { + break + } + // The selected branch starts with this rune, so consume it and jump + // directly to the rest of that branch. + r.Runtextpos += r.bump() + if err := r.goTo(table.Branches[branch]); err != nil { + return err + } continue case syntax.Testref: @@ -248,7 +322,9 @@ func executeDefault(r *Runner) error { case syntax.Lazybranch | syntax.Back: r.trackPop() r.textto(r.trackPeek()) - r.goTo(r.operand(0)) + if err := r.goTo(r.operand(0)); err != nil { + return err + } continue case syntax.Setmark: @@ -307,9 +383,11 @@ func executeDefault(r *Runner) error { matched := r.textPos() - r.stackPeek() if matched != 0 { // Nonempty match -> loop now - r.trackPush2(r.stackPeek(), r.textPos()) // Save old mark, textpos - r.stackPush(r.textPos()) // Make new mark - r.goTo(r.operand(0)) // Loop + r.trackPush2(r.stackPeek(), r.textPos()) // Save old mark, textpos + r.stackPush(r.textPos()) // Make new mark + if err := r.goTo(r.operand(0)); err != nil { // Loop + return err + } } else { // Empty match -> straight now r.trackPushNeg1(r.stackPeek()) // Save old mark r.advance(1) // Straight @@ -364,10 +442,12 @@ func executeDefault(r *Runner) error { r.trackPopN(2) pos := r.trackPeekN(1) - r.trackPushNeg2(r.trackPeek(), 1) // Save old mark, note that we pushed a new mark - r.stackPush(pos) // Make new mark - r.textto(pos) // Recall position - r.goTo(r.operand(0)) // Loop + r.trackPushNeg2(r.trackPeek(), 1) // Save old mark, note that we pushed a new mark + r.stackPush(pos) // Make new mark + r.textto(pos) // Recall position + if err := r.goTo(r.operand(0)); err != nil { // Loop + return err + } continue case syntax.Lazybranchmark | syntax.Back2: @@ -413,9 +493,11 @@ func executeDefault(r *Runner) error { r.trackPushNeg2(mark, count) // Save old mark, count r.advance(2) // Straight } else { // Nonempty match -> count+loop now - r.trackPush1(mark) // remember mark - r.stackPush2(r.textPos(), count+1) // Make new mark, incr count - r.goTo(r.operand(0)) // Loop + r.trackPush1(mark) // remember mark + r.stackPush2(r.textPos(), count+1) // Make new mark, incr count + if err := r.goTo(r.operand(0)); err != nil { // Loop + return err + } } continue @@ -452,9 +534,11 @@ func executeDefault(r *Runner) error { count := r.stackPeekN(1) if count < 0 { // Negative count -> loop now - r.trackPushNeg1(mark) // Save old mark - r.stackPush2(r.textPos(), count+1) // Make new mark, incr count - r.goTo(r.operand(0)) // Loop + r.trackPushNeg1(mark) // Save old mark + r.stackPush2(r.textPos(), count+1) // Make new mark, incr count + if err := r.goTo(r.operand(0)); err != nil { // Loop + return err + } } else { // Nonneg count -> straight now r.trackPush3(mark, count, r.textPos()) // Save mark, count, position r.advance(2) // Straight @@ -472,10 +556,12 @@ func executeDefault(r *Runner) error { textpos := r.trackPeekN(2) if r.trackPeekN(1) < r.operand(1) && textpos != mark { // Under limit and not empty match -> loop - r.textto(textpos) // Recall position - r.stackPush2(textpos, r.trackPeekN(1)+1) // Make new mark, incr count - r.trackPushNeg1(mark) // Save old mark - r.goTo(r.operand(0)) // Loop + r.textto(textpos) // Recall position + r.stackPush2(textpos, r.trackPeekN(1)+1) // Make new mark, incr count + r.trackPushNeg1(mark) // Save old mark + if err := r.goTo(r.operand(0)); err != nil { // Loop + return err + } continue } else { // Max loops or empty match -> backtrack r.stackPush2(r.trackPeek(), r.trackPeekN(1)) // Recall old mark, count @@ -648,6 +734,13 @@ func executeDefault(r *Runner) error { r.advance(1) continue + case syntax.Grapheme: + if !r.TryMatchGrapheme(r.rightToLeft) { + break + } + r.advance(0) + continue + case syntax.Ref: capnum := r.operand(0) @@ -935,18 +1028,21 @@ func executeDefault(r *Runner) error { ; // "break Backward" comes here: - r.backtrack() + if err := r.backtrack(); err != nil { + return err + } } } // increase the size of stack and track storage -func (r *Runner) ensureStorage() { +func (r *Runner) ensureStorage() error { if r.Runstackpos < r.runtrackcount*4 { doubleIntSlice(&r.runstack, &r.Runstackpos) } - if r.Runtrackpos < r.runtrackcount*4 { - doubleIntSlice(&r.runtrack, &r.Runtrackpos) + if r.Runtrackpos < r.runtrackcount*4 && !r.growTrack() { + return ErrBacktrackingStackLimit } + return nil } func (r *Runner) ensureStack(plus int) { @@ -990,14 +1086,17 @@ func (r *Runner) advance(i int) { r.setOperator(r.code.Codes[r.codepos]) } -func (r *Runner) goTo(newpos int) { +func (r *Runner) goTo(newpos int) error { // when branching backward or in place, ensure storage if newpos <= r.codepos { - r.ensureStorage() + if err := r.ensureStorage(); err != nil { + return err + } } r.setOperator(r.code.Codes[newpos]) r.codepos = newpos + return nil } func (r *Runner) textto(newpos int) { @@ -1016,11 +1115,47 @@ func (r *Runner) textPos() int { return r.Runtextpos } +// TryMatchGrapheme consumes one Unicode extended grapheme cluster in the +// runner's current direction. It is exported for regexp2cg-generated engines. +func (r *Runner) TryMatchGrapheme(rightToLeft bool) bool { + var boundary int + if rightToLeft { + boundary = syntax.PreviousGraphemeClusterBoundary(r.Runtext, r.Runtextpos) + } else { + boundary = syntax.NextGraphemeClusterBoundary(r.Runtext, r.Runtextpos) + } + if boundary < 0 { + return false + } + r.Runtextpos = boundary + return true +} + // push onto the backtracking stack func (r *Runner) trackpos() int { return len(r.runtrack) - r.Runtrackpos } +func (r *Runner) growTrack() bool { + oldLen := len(r.runtrack) + newLen := oldLen * 2 + if newLen == 0 { + newLen = 1 + } + if limit := r.re.optimizations.MaxBacktrackingStackSize; limit >= 0 && newLen > limit { + newLen = limit + } + if newLen <= oldLen { + return false + } + + newTrack := make([]int, newLen) + copy(newTrack[newLen-oldLen:], r.runtrack) + r.Runtrackpos += newLen - oldLen + r.runtrack = newTrack + return true +} + func (r *Runner) trackPush() { r.Runtrackpos-- r.runtrack[r.Runtrackpos] = r.codepos @@ -1069,7 +1204,7 @@ func (r *Runner) trackPushNeg2(I1, I2 int) { r.runtrack[r.Runtrackpos] = -r.codepos } -func (r *Runner) backtrack() { +func (r *Runner) backtrack() error { newpos := r.runtrack[r.Runtrackpos] r.Runtrackpos++ @@ -1090,10 +1225,13 @@ func (r *Runner) backtrack() { // When branching backward, ensure storage if newpos < r.codepos { - r.ensureStorage() + if err := r.ensureStorage(); err != nil { + return err + } } r.codepos = newpos + return nil } func (r *Runner) setOperator(op int) { @@ -1397,7 +1535,8 @@ func shouldUseFindFirstCharOptimized(r *Runner) bool { return false } - switch r.code.FindOptimizations.FindMode { + opts := r.code.FindOptimizations + switch opts.FindMode { case syntax.TrailingAnchor_FixedLength_LeftToRight_End, syntax.LeadingString_OrdinalIgnoreCase_LeftToRight, syntax.LeadingStrings_LeftToRight, @@ -1408,6 +1547,13 @@ func shouldUseFindFirstCharOptimized(r *Runner) bool { syntax.LiteralAfterLoop_LeftToRight, syntax.RequiredLandmarkChain_LeftToRight: return true + case syntax.LeadingSet_LeftToRight: + // General Unicode sets already have a direct fallback loop below. + // Large enumerated sets are also faster through the set's ASCII bitmap + // than through the linear IndexOfAny helper. + return len(opts.FixedDistanceSets) > 0 && + ((len(opts.FixedDistanceSets[0].Chars) > 0 && len(opts.FixedDistanceSets[0].Chars) <= 5) || + opts.FixedDistanceSets[0].Range != nil) default: return false } @@ -1429,10 +1575,10 @@ func findFirstCharOptimized(r *Runner) (handled bool, found bool) { case syntax.LeadingString_OrdinalIgnoreCase_LeftToRight: return true, findLeadingStringLeftToRight(r, []rune(opts.LeadingPrefix), true) case syntax.LeadingStrings_LeftToRight: - return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, false) + return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, opts.LeadingPrefixFirstRunes, false) case syntax.LeadingStrings_OrdinalIgnoreCase_LeftToRight: - return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, true) - case syntax.FixedDistanceSets_LeftToRight: + return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, opts.LeadingPrefixFirstRunes, true) + case syntax.LeadingSet_LeftToRight, syntax.FixedDistanceSets_LeftToRight: return true, findFixedDistanceSetsLeftToRight(r, opts.FixedDistanceSets) case syntax.FixedDistanceChar_LeftToRight: return true, findFixedDistanceCharLeftToRight(r, opts.FixedDistanceLiteral.C, opts.FixedDistanceLiteral.Distance) @@ -1487,29 +1633,69 @@ func findLeadingStringLeftToRight(r *Runner, prefix []rune, ignoreCase bool) boo return true } -func findLeadingStringsLeftToRight(r *Runner, prefixes [][]rune, ignoreCase bool) bool { +func findLeadingStringsLeftToRight(r *Runner, prefixes [][]rune, firstRunes []rune, ignoreCase bool) bool { if len(prefixes) == 0 { return false } - for start := r.Runtextpos; start <= latestPossibleStart(r); start++ { - for _, prefix := range prefixes { - if ignoreCase { - if helpers.StartsWithIgnoreCase(r.Runtext[start:], prefix) { + // Unicode ordinal-ignore-case matching has more possible first-rune folds + // than a small precomputed set can safely represent. Keep its conservative + // position-by-position scan; the common case-sensitive path skips directly + // between possible first runes. + if ignoreCase || len(firstRunes) == 0 { + for start := r.Runtextpos; start <= latestPossibleStart(r); start++ { + for _, prefix := range prefixes { + if ignoreCase { + if helpers.StartsWithIgnoreCase(r.Runtext[start:], prefix) { + r.Runtextpos = start + return true + } + } else if helpers.StartsWith(r.Runtext[start:], prefix) { r.Runtextpos = start return true } - } else if helpers.StartsWith(r.Runtext[start:], prefix) { + } + } + r.Runtextpos = r.Runtextend + return false + } + + latest := min(latestPossibleStart(r), r.Runtextend-1) + for searchAt := r.Runtextpos; searchAt <= latest; { + offset := indexOfAnyRunes(r.Runtext[searchAt:latest+1], firstRunes) + if offset < 0 { + break + } + start := searchAt + offset + first := r.Runtext[start] + for _, prefix := range prefixes { + if len(prefix) > 0 && prefix[0] == first && helpers.StartsWith(r.Runtext[start:], prefix) { r.Runtextpos = start return true } } + searchAt = start + 1 } r.Runtextpos = r.Runtextend return false } +func indexOfAnyRunes(input, find []rune) int { + switch len(find) { + case 0: + return -1 + case 1: + return helpers.IndexOfAny1(input, find[0]) + case 2: + return helpers.IndexOfAny2(input, find[0], find[1]) + case 3: + return helpers.IndexOfAny3(input, find[0], find[1], find[2]) + default: + return helpers.IndexOfAny(input, find) + } +} + func findFixedDistanceCharLeftToRight(r *Runner, ch rune, distance int) bool { searchStart := r.Runtextpos + distance for searchStart < r.Runtextend { @@ -1850,6 +2036,9 @@ func (r *Runner) initMatch(textInfo *matchText) { if tracksize < 64 { tracksize = 64 } + if limit := r.re.optimizations.MaxBacktrackingStackSize; limit >= 0 && tracksize > limit { + tracksize = limit + } if stacksize < 32 { stacksize = 32 } @@ -1882,9 +2071,7 @@ func (r *Runner) tidyMatch(quick bool) *Match { m.textpos = r.Runtextpos if m.matchcount[0] > 0 { interval := m.matches[0] - // bytes indices aren't used so just use fast path - m.RuneIndex = interval[0] - m.RuneLength = interval[1] + setCaptureFields(&m.Capture, interval[0], interval[1]) } return m } @@ -2056,36 +2243,6 @@ func (r *Runner) initTrackCount() { } } -// decodeString converts s to []rune using a shared size-classed buffer pool when -// allowed by the regexp optimization settings. Pooled slices must be returned -// after the runner is done with them. -func (r *Runner) decodeString(s string) ([]rune, *[]rune) { - buf, pooled := pooledRuneBuffers.get(len(s), r.re.optimizations.MaxCachedRuneBufferLength) - n := 0 - for _, ch := range s { - buf[n] = ch - n++ - } - return buf[:n], pooled -} - -func (r *Runner) decodeStringWithStart(s string, startAt int) (runes []rune, runeStart int, pooled *[]rune) { - buf, pooled := pooledRuneBuffers.get(len(s), r.re.optimizations.MaxCachedRuneBufferLength) - n := 0 - runeStart = -1 - for strIdx, ch := range s { - if startAt >= 0 && strIdx == startAt { - runeStart = n - } - buf[n] = ch - n++ - } - if startAt >= 0 && startAt == len(s) { - runeStart = n - } - return buf[:n], runeStart, pooled -} - // getRunner returns a runner to use for matching re. func (re *Regexp) getRunner() *Runner { if re.runnerPool == nil { @@ -2097,6 +2254,7 @@ func (re *Regexp) getRunner() *Runner { // putRunner returns a runner to the re's pool cache. func (re *Regexp) putRunner(r *Runner) { r.Runtext = nil + r.code = re.code if r.runmatch != nil { r.runmatch.text = nil } @@ -2128,6 +2286,12 @@ func (r *Runner) StackPop() int { return val } +// StackDepth returns the number of integer slots currently used by the +// generated engine's backtracking stack. +func (r *Runner) StackDepth() int { + return len(r.runstack) - r.Runstackpos +} + func (r *Runner) StackPush(val int) { // check if we need to size up stack r.ensureStack(1) diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/split.go b/backend/vendor/github.com/dlclark/regexp2/v2/split.go index 1dbe0dcd..3c161988 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/split.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/split.go @@ -33,21 +33,20 @@ func (re *Regexp) Split(input string, count int) ([]string, error) { // iterate through the matches priorIndex := 0 var retVal []string - var txt []rune + matched := false m, err := re.FindStringMatch(input) for ; m != nil && count > 0; m, err = re.FindNextMatch(m) { - txt = m.text.runes - // if we have an m, we don't have an err - // append our match - retVal = append(retVal, string(txt[priorIndex:m.RuneIndex])) + matched = true + start, end := matchInputSpan(m) + retVal = append(retVal, input[priorIndex:start]) // append any capture groups, skipping group 0 gs := m.Groups() for i := 1; i < len(gs); i++ { retVal = append(retVal, gs[i].String()) } - priorIndex = m.RuneIndex + m.RuneLength + priorIndex = end count-- } @@ -55,13 +54,10 @@ func (re *Regexp) Split(input string, count int) ([]string, error) { return nil, err } - if txt == nil { - // we never matched, return the original string + if !matched { return []string{input}, nil } - // append our remainder - retVal = append(retVal, string(txt[priorIndex:])) - + retVal = append(retVal, input[priorIndex:]) return retVal, nil } diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/stringprefixfilter.go b/backend/vendor/github.com/dlclark/regexp2/v2/stringprefixfilter.go index fe9cb6ad..080b8c29 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/stringprefixfilter.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/stringprefixfilter.go @@ -39,6 +39,15 @@ func newStringPrefixFilter(code *syntax.Code) StringPrefixFilter { return stringIndexPrefixesFilter(opts.LeadingPrefixes, false, minRequiredLength) case syntax.LeadingStrings_OrdinalIgnoreCase_LeftToRight: return stringIndexPrefixesFilter(opts.LeadingPrefixes, true, minRequiredLength) + case syntax.LeadingSet_LeftToRight: + if len(opts.FixedDistanceSets) == 0 { + return nil + } + set := opts.FixedDistanceSets[0] + if set.Range == nil && (len(set.Chars) == 0 || len(set.Chars) > 5) { + return nil + } + return stringFixedDistanceSetFilter(set, minRequiredLength) case syntax.FixedDistanceChar_LeftToRight: return stringFixedDistanceCharFilter(opts.FixedDistanceLiteral.C, opts.FixedDistanceLiteral.Distance, minRequiredLength) case syntax.FixedDistanceString_LeftToRight: @@ -50,6 +59,87 @@ func newStringPrefixFilter(code *syntax.Code) StringPrefixFilter { } } +type asciiSetStringScanner struct { + chars string + first byte + last byte + useRange bool + distance int +} + +func newASCIISetStringScanner(set syntax.FixedDistanceSet) (asciiSetStringScanner, bool) { + if set.Negated || set.Distance < 0 { + return asciiSetStringScanner{}, false + } + if set.Range != nil { + if set.Range.First < 0 || set.Range.Last > utf8.RuneSelf-1 { + return asciiSetStringScanner{}, false + } + return asciiSetStringScanner{ + first: byte(set.Range.First), + last: byte(set.Range.Last), + useRange: true, + distance: set.Distance, + }, true + } + if len(set.Chars) == 0 { + return asciiSetStringScanner{}, false + } + chars := make([]byte, len(set.Chars)) + for i, ch := range set.Chars { + if ch < 0 || ch > utf8.RuneSelf-1 { + return asciiSetStringScanner{}, false + } + chars[i] = byte(ch) + } + return asciiSetStringScanner{chars: string(chars), distance: set.Distance}, true +} + +func stringFixedDistanceSetFilter(set syntax.FixedDistanceSet, minRequiredLength int) StringPrefixFilter { + scanner, ok := newASCIISetStringScanner(set) + if !ok { + return nil + } + + return func(input string, startAt int) (candidateByteIndex int, ok bool) { + if !hasMinRequiredBytes(input, startAt, minRequiredLength) { + return 0, false + } + + for searchAt := startAt; searchAt < len(input); { + offset := scanner.index(input[searchAt:]) + if offset < 0 { + return 0, false + } + setByteIndex := searchAt + offset + candidateByteIndex, valid := stringFixedDistanceCandidateStart(input, startAt, setByteIndex, scanner.distance) + if valid && hasMinRequiredBytes(input, candidateByteIndex, minRequiredLength) { + return candidateByteIndex, true + } + if valid { + return 0, false + } + searchAt = setByteIndex + 1 + } + return 0, false + } +} + +func (s asciiSetStringScanner) index(input string) int { + if !s.useRange { + if len(s.chars) == 1 { + return strings.IndexByte(input, s.chars[0]) + } + return strings.IndexAny(input, s.chars) + } + for i := 0; i < len(input); i++ { + if input[i] >= s.first && input[i] <= s.last { + return i + } + } + return -1 +} + func stringIndexPrefixFilter(prefix string, ignoreCase bool, minRequiredLength int) StringPrefixFilter { if prefix == "" { return nil diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/code.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/code.go index 1df4e45a..69cad966 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/code.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/code.go @@ -91,6 +91,11 @@ const ( Setloopatomic InstOp = 45 // Updates the bumpalong position to the current position. UpdateBumpalong InstOp = 46 + // Matches one Unicode extended grapheme cluster (\X). + Grapheme InstOp = 47 + // Selects and consumes the next character using a disjoint branch table. + // Operand 0 is an index into Code.Dispatches. + Dispatch InstOp = 48 // Modifiers for alternate modes @@ -105,6 +110,7 @@ type Code struct { Codes []int // the code Strings [][]rune // string table Sets []*CharSet //character set table + Dispatches []DispatchTable // shared match tables for deterministic alternations TrackCount int // how many instructions use backtracking Caps map[int]int // mapping of user group numbers -> impl group slots Capsize int // number of impl group slots @@ -113,6 +119,56 @@ type Code struct { Anchors AnchorLoc // the set of zero-length start anchors (RegexFCD.Bol, etc) RightToLeft bool // true if right to left FindOptimizations *FindOptimizations // analyzed candidate search strategy + QuickCodes []int // bool-only code with unobservable captures removed + QuickDispatches []DispatchTable // lightweight tables targeting QuickCodes + CaptureSlotInUse []bool // capture slots observable by the pattern itself during quick matches + // LeftContextRunes is how many runes before a candidate start matching may + // inspect. 0 means none, 1 means a single previous rune (or slack so ^/\A + // do not see the candidate as the origin), and -1 means do not slice + // (lookbehind or \G). + LeftContextRunes int +} + +// DispatchTable maps disjoint character sets to branch indices. Larger tables +// use a direct ASCII lookup, small tables use compact masks, and non-ASCII +// runes inspect the full sets. +type DispatchTable struct { + Sets []int + Branches []int + ASCII *[128]uint16 // entry index + 1; zero means no branch + ASCIIMasks []uint64 // two membership words per branch when ASCII is nil +} + +// captureSlotsInUse returns the capture slots whose values can affect matching. +// Group 0 is always retained as the success marker. Ordinary captures that are +// never referenced by the pattern may be omitted by bool-only matching APIs. +func captureSlotsInUse(codes []int, capsize int) []bool { + inUse := make([]bool, capsize) + if capsize > 0 { + inUse[0] = true + } + for pos := 0; pos < len(codes); { + op := InstOp(codes[pos]) & Mask + switch op { + case Ref, Testref: + capnum := codes[pos+1] + if capnum >= 0 && capnum < len(inUse) { + inUse[capnum] = true + } + case Capturemark: + // Balancing groups both observe and mutate capture state. Keep both + // sides live even if no later backreference refers to them. + if codes[pos+2] != -1 { + for _, capnum := range codes[pos+1 : pos+3] { + if capnum >= 0 && capnum < len(inUse) { + inUse[capnum] = true + } + } + } + } + pos += opcodeSize(op) + } + return inUse } // PrepareCharSetASCIIBitmaps builds bounded ASCII lookup tables for compiled @@ -156,11 +212,11 @@ func opcodeSize(op InstOp) int { switch op { case Nothing, Bol, Eol, Boundary, Nonboundary, ECMABoundary, NonECMABoundary, Beginning, Start, EndZ, - End, Nullmark, Setmark, Getmark, Setjump, Backjump, Forejump, Stop, UpdateBumpalong: + End, Nullmark, Setmark, Getmark, Setjump, Backjump, Forejump, Stop, UpdateBumpalong, Grapheme: return 1 case One, Notone, Multi, Ref, Testref, Goto, Nullcount, Setcount, Lazybranch, Branchmark, Lazybranchmark, - Prune, Set: + Prune, Set, Dispatch: return 2 case Capturemark, Branchcount, Lazybranchcount, Onerep, Notonerep, Oneloop, Notoneloop, Onelazy, Notonelazy, @@ -187,7 +243,7 @@ var codeStr = []string{ "Prune", "Stop", "ECMABoundary", "NonECMABoundary", "Oneloopatomic", "Notoneloopatomic", "Setloopatomic", - "Bumpalong", + "Bumpalong", "Grapheme", "Dispatch", } func operatorDescription(op InstOp) string { @@ -234,6 +290,16 @@ func (c *Code) OpcodeDescription(offset int) string { buf.WriteString("Set = ") buf.WriteString(c.Sets[c.Codes[offset+1]].String()) + case Dispatch: + tableIndex := c.Codes[offset+1] + fmt.Fprintf(buf, "Table = %d", tableIndex) + if tableIndex >= 0 && tableIndex < len(c.Dispatches) { + table := &c.Dispatches[tableIndex] + for i, setIndex := range table.Sets { + fmt.Fprintf(buf, ", %s -> %d", c.Sets[setIndex].String(), table.Branches[i]) + } + } + case Multi: fmt.Fprintf(buf, "String = %s", string(c.Strings[c.Codes[offset+1]])) diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/grapheme.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/grapheme.go new file mode 100644 index 00000000..c4faf355 --- /dev/null +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/grapheme.go @@ -0,0 +1,294 @@ +package syntax + +import "unicode" + +type graphemeBreakClass uint8 + +const ( + graphemeOther graphemeBreakClass = iota + graphemeCR + graphemeLF + graphemeControl + graphemeExtend + graphemeZWJ + graphemeRegionalIndicator + graphemePrepend + graphemeSpacingMark + graphemeL + graphemeV + graphemeT + graphemeLV + graphemeLVT +) + +type indicConjunctBreakClass uint8 + +const ( + indicNone indicConjunctBreakClass = iota + indicConsonant + indicExtend + indicLinker +) + +type graphemeProperties struct { + breakClass graphemeBreakClass + indicClass indicConjunctBreakClass + extendedPictographic bool +} + +// NextGraphemeClusterBoundary returns the first extended-grapheme boundary +// after start, or -1 when start is not a valid input position. It implements +// the Unicode 17 rules from UAX #29 with a small state machine, avoiding the +// backtracking and repeated character-class probes of an equivalent regexp. +func NextGraphemeClusterBoundary(text []rune, start int) int { + if start < 0 || start >= len(text) { + return -1 + } + + previous := graphemePropertiesFor(text[start]) + state := graphemeForwardState{} + state.consume(previous) + + for pos := start + 1; pos < len(text); pos++ { + current := graphemePropertiesFor(text[pos]) + if isGraphemeBoundaryForward(previous.breakClass, current, state) { + return pos + } + state.consume(current) + previous = current + } + return len(text) +} + +// PreviousGraphemeClusterBoundary returns the first extended-grapheme boundary +// before end, or -1 when end is not a valid input position. +func PreviousGraphemeClusterBoundary(text []rune, end int) int { + if end <= 0 || end > len(text) { + return -1 + } + for pos := end - 1; pos > 0; pos-- { + if isGraphemeBoundary(text, pos) { + return pos + } + } + return 0 +} + +type graphemeForwardState struct { + regionalIndicatorCount int + extendedPictographicRun bool + zwjAfterPictographic bool + indicConsonantRun bool + indicLinkerSeen bool +} + +func (s *graphemeForwardState) consume(properties graphemeProperties) { + if properties.breakClass == graphemeRegionalIndicator { + s.regionalIndicatorCount++ + } else { + s.regionalIndicatorCount = 0 + } + + if properties.breakClass == graphemeZWJ { + s.zwjAfterPictographic = s.extendedPictographicRun + } else { + s.zwjAfterPictographic = false + } + if properties.extendedPictographic { + s.extendedPictographicRun = true + } else if properties.breakClass != graphemeExtend { + s.extendedPictographicRun = false + } + + switch properties.indicClass { + case indicConsonant: + s.indicConsonantRun = true + s.indicLinkerSeen = false + case indicExtend: + // Extend preserves a preceding consonant/linker run. + case indicLinker: + if s.indicConsonantRun { + s.indicLinkerSeen = true + } + default: + s.indicConsonantRun = false + s.indicLinkerSeen = false + } +} + +func isGraphemeBoundaryForward(previous graphemeBreakClass, current graphemeProperties, state graphemeForwardState) bool { + if previous == graphemeCR && current.breakClass == graphemeLF { // GB3 + return false + } + if isGraphemeControl(previous) || isGraphemeControl(current.breakClass) { // GB4, GB5 + return true + } + if hangulNoBreak(previous, current.breakClass) { // GB6, GB7, GB8 + return false + } + if current.breakClass == graphemeExtend || current.breakClass == graphemeZWJ || current.breakClass == graphemeSpacingMark { // GB9, GB9a + return false + } + if previous == graphemePrepend { // GB9b + return false + } + if current.indicClass == indicConsonant && state.indicConsonantRun && state.indicLinkerSeen { // GB9c + return false + } + if current.extendedPictographic && state.zwjAfterPictographic { // GB11 + return false + } + if previous == graphemeRegionalIndicator && current.breakClass == graphemeRegionalIndicator && state.regionalIndicatorCount%2 == 1 { // GB12, GB13 + return false + } + return true +} + +func isGraphemeBoundary(text []rune, pos int) bool { + previous := graphemePropertiesFor(text[pos-1]) + current := graphemePropertiesFor(text[pos]) + if previous.breakClass == graphemeCR && current.breakClass == graphemeLF { // GB3 + return false + } + if isGraphemeControl(previous.breakClass) || isGraphemeControl(current.breakClass) { // GB4, GB5 + return true + } + if hangulNoBreak(previous.breakClass, current.breakClass) { // GB6, GB7, GB8 + return false + } + if current.breakClass == graphemeExtend || current.breakClass == graphemeZWJ || current.breakClass == graphemeSpacingMark { // GB9, GB9a + return false + } + if previous.breakClass == graphemePrepend { // GB9b + return false + } + if current.indicClass == indicConsonant && hasIndicConjunctBefore(text, pos) { // GB9c + return false + } + if current.extendedPictographic && hasExtendedPictographicZWJBefore(text, pos) { // GB11 + return false + } + if previous.breakClass == graphemeRegionalIndicator && current.breakClass == graphemeRegionalIndicator { // GB12, GB13 + count := 0 + for i := pos - 1; i >= 0 && graphemeClass(text[i]) == graphemeRegionalIndicator; i-- { + count++ + } + return count%2 == 0 + } + return true +} + +func hasIndicConjunctBefore(text []rune, pos int) bool { + linkerSeen := false + for i := pos - 1; i >= 0; i-- { + switch indicClass(text[i]) { + case indicExtend: + continue + case indicLinker: + linkerSeen = true + continue + case indicConsonant: + return linkerSeen + default: + return false + } + } + return false +} + +func hasExtendedPictographicZWJBefore(text []rune, pos int) bool { + i := pos - 1 + if i < 0 || graphemeClass(text[i]) != graphemeZWJ { + return false + } + for i--; i >= 0 && graphemeClass(text[i]) == graphemeExtend; i-- { + } + return i >= 0 && unicode.Is(unicodeAliasExtended_Pictographic, text[i]) +} + +func isGraphemeControl(class graphemeBreakClass) bool { + return class == graphemeCR || class == graphemeLF || class == graphemeControl +} + +func hangulNoBreak(previous, current graphemeBreakClass) bool { + return previous == graphemeL && (current == graphemeL || current == graphemeV || current == graphemeLV || current == graphemeLVT) || + (previous == graphemeLV || previous == graphemeV) && (current == graphemeV || current == graphemeT) || + (previous == graphemeLVT || previous == graphemeT) && current == graphemeT +} + +func indicClass(ch rune) indicConjunctBreakClass { + if ch < 0x300 { + return indicNone + } + if unicode.Is(unicodeAliasIndic_Conjunct_Break_Consonant, ch) { + return indicConsonant + } + if unicode.Is(unicodeAliasIndic_Conjunct_Break_Linker, ch) { + return indicLinker + } + if unicode.Is(unicodeAliasIndic_Conjunct_Break_Extend, ch) { + return indicExtend + } + return indicNone +} + +func graphemePropertiesFor(ch rune) graphemeProperties { + if ch <= unicode.MaxASCII { + return graphemeProperties{breakClass: graphemeClass(ch)} + } + return graphemeProperties{ + breakClass: graphemeClass(ch), + indicClass: indicClass(ch), + extendedPictographic: ch >= 0xA9 && unicode.Is(unicodeAliasExtended_Pictographic, ch), + } +} + +func graphemeClass(ch rune) graphemeBreakClass { + if ch <= unicode.MaxASCII { + switch ch { + case '\r': + return graphemeCR + case '\n': + return graphemeLF + default: + if ch < ' ' || ch == 0x7F { + return graphemeControl + } + return graphemeOther + } + } + if ch == 0x200D { + return graphemeZWJ + } + if ch >= 0x1F1E6 && ch <= 0x1F1FF { + return graphemeRegionalIndicator + } + if ch >= 0xAC00 && ch <= 0xD7A3 { + if (ch-0xAC00)%28 == 0 { + return graphemeLV + } + return graphemeLVT + } + if ch >= 0x1100 && ch <= 0x115F || ch >= 0xA960 && ch <= 0xA97C { + return graphemeL + } + if ch >= 0x1160 && ch <= 0x11A7 || ch >= 0xD7B0 && ch <= 0xD7C6 { + return graphemeV + } + if ch >= 0x11A8 && ch <= 0x11FF || ch >= 0xD7CB && ch <= 0xD7FB { + return graphemeT + } + if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Control, ch) { + return graphemeControl + } + if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Extend, ch) { + return graphemeExtend + } + if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Prepend, ch) { + return graphemePrepend + } + if unicode.Is(unicodeAliasGrapheme_Cluster_Break_SpacingMark, ch) { + return graphemeSpacingMark + } + return graphemeOther +} diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/optimizations.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/optimizations.go index 0186497a..67d77539 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/optimizations.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/optimizations.go @@ -21,10 +21,11 @@ type FindOptimizations struct { LeadingPrefixesRunes [][]rune //LeadingStrings *helpers.StringSearchValues - FixedDistanceLiteral FixedDistanceLiteral - FixedDistanceSets []FixedDistanceSet - LiteralAfterLoop *LiteralAfterLoop - LandmarkChain *RequiredLandmarkChain + FixedDistanceLiteral FixedDistanceLiteral + FixedDistanceSets []FixedDistanceSet + LiteralAfterLoop *LiteralAfterLoop + LandmarkChain *RequiredLandmarkChain + LeadingPrefixFirstRunes []rune } type LiteralAfterLoop struct { @@ -460,19 +461,23 @@ func newFindOptimizationsForNode(root *RegexNode, opt ParseOptions, isLeadingPar // We're now left-to-right only and looking for multiple prefixes and/or sets. - // If there are multiple leading strings, we can search for any of them. - // this works in the interpreter, but we avoid it due to additional cost during construction - + // Multiple leading strings let the finder jump between candidate prefixes. + // Case-sensitive prefixes are cheap to collect and help the interpreter. + // Case-insensitive prefix explosion is still limited to code generation. + if prefixes := findPrefixes(root, false); len(prefixes) > 1 { + f.LeadingPrefixes = prefixes + f.LeadingPrefixesRunes = toRunePrefixes(prefixes) + f.LeadingPrefixFirstRunes = leadingPrefixFirstRunes(f.LeadingPrefixesRunes) + f.FindMode = LeadingStrings_LeftToRight + return f + } if !interpreter { ciPrefixes := findPrefixes(root, true) if len(ciPrefixes) > 1 { f.LeadingPrefixes = ciPrefixes f.LeadingPrefixesRunes = toRunePrefixes(ciPrefixes) + f.LeadingPrefixFirstRunes = leadingPrefixFirstRunes(f.LeadingPrefixesRunes) f.FindMode = LeadingStrings_OrdinalIgnoreCase_LeftToRight - /*SYSTEM_TEXT_REGULAREXPRESSIONS - if usesRfoTryFind { - f.LeadingStrings = helpers.NewSearchValues(f.LeadingPrefixes, true) - }*/ return f } } @@ -515,19 +520,6 @@ func newFindOptimizationsForNode(root *RegexNode, opt ParseOptions, isLeadingPar // In some searches, we may use multiple sets, so we want the subsequent ones to also be the efficiency runners-up. slices.SortFunc(fixedDistanceSets, compareFixedDistanceSetsByQuality) - // If the best fixed-distance set is composed of high-frequency characters, IndexOfAny on - // those characters is likely to match too many positions. Prefer a case-sensitive - // multi-prefix search when one is available. - if !interpreter && !mayContainCaseInsensitiveMatching(root) && hasHighFrequencyChars(fixedDistanceSets[0]) { - caseSensitivePrefixes := findPrefixes(root, false) - if len(caseSensitivePrefixes) > 1 { - f.LeadingPrefixes = caseSensitivePrefixes - f.LeadingPrefixesRunes = toRunePrefixes(caseSensitivePrefixes) - f.FindMode = LeadingStrings_LeftToRight - return f - } - } - // If there is no literal after the loop, use whatever set we got. // If there is a literal after the loop, consider it to be better than a negated set and better than a set with many characters. if literalAfterLoop == nil || (len(fixedDistanceSets[0].Chars) > 0 && !fixedDistanceSets[0].Negated) { @@ -587,6 +579,16 @@ func toRunePrefixes(prefixes []string) [][]rune { return runes } +func leadingPrefixFirstRunes(prefixes [][]rune) []rune { + first := make([]rune, 0, len(prefixes)) + for _, prefix := range prefixes { + if len(prefix) > 0 && !slices.Contains(first, prefix[0]) { + first = append(first, prefix[0]) + } + } + return first +} + func getFindMode(rtl bool, t NodeType) FindNextStartingPositionMode { if rtl { switch t { diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/parser.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/parser.go index 9e9029a8..0e842828 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/parser.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/parser.go @@ -636,11 +636,16 @@ func (p *parser) scanRegex() (*RegexNode, error) { } case '\\': + quoted := !p.useOptionE() && p.charsRight() > 0 && p.rightChar(0) == 'Q' n, err := p.scanBackslash(false) if err != nil { return nil, err } - p.addUnitNode(n) + if quoted { + p.addQuotedUnit(n) + } else { + p.addUnitNode(n) + } case '^': if p.useOptionM() { @@ -702,7 +707,7 @@ func (p *parser) scanRegex() (*RegexNode, error) { // Handle quantifiers for p.unit != nil { var min, max int - var lazy bool + var lazy, possessive bool switch ch { case '*': @@ -753,18 +758,19 @@ func (p *parser) scanRegex() (*RegexNode, error) { return nil, err } - if p.charsRight() == 0 || p.rightChar(0) != '?' { - lazy = false - } else { + if p.charsRight() > 0 && p.rightChar(0) == '?' { p.moveRight(1) lazy = true + } else if p.charsRight() > 0 && p.rightChar(0) == '+' && !p.useOptionE() { + p.moveRight(1) + possessive = true } if min > max { return nil, p.getErr(ErrInvalidRepeatSize) } - p.addConcatenate3(lazy, min, max) + p.addConcatenate3(lazy, possessive, min, max) } ContinueOuterScan: @@ -1280,6 +1286,37 @@ func (p *parser) scanBackslash(scanOnly bool) (*RegexNode, error) { } switch ch := p.rightChar(0); ch { + case 'Q': + if p.useOptionE() { + return p.scanBasicBackslash(scanOnly) + } + p.moveRight(1) + quoted := p.scanQuoted() + if scanOnly { + return nil, nil + } + return newRegexNodeStr(NtMulti, p.options&^IgnoreCase, quoted), nil + + case 'R': + if p.useOptionE() || p.useRE2() { + return p.scanBasicBackslash(scanOnly) + } + p.moveRight(1) + if scanOnly { + return nil, nil + } + return newUnicodeNewlineNode(p.options), nil + + case 'X': + if p.useOptionE() || p.useRE2() { + return p.scanBasicBackslash(scanOnly) + } + p.moveRight(1) + if scanOnly { + return nil, nil + } + return newRegexNode(NtGrapheme, p.options), nil + case 'b', 'B', 'A', 'G', 'Z', 'z': p.moveRight(1) return newRegexNode(p.typeFromCode(ch), p.options), nil @@ -1354,6 +1391,58 @@ func (p *parser) scanBackslash(scanOnly bool) (*RegexNode, error) { } } +// newUnicodeNewlineNode constructs \R from existing general-purpose nodes. +// Its branches have disjoint starting sets, and the optional LF is atomic, so +// reduction can remove the outer atomic wrapper without changing semantics. +func newUnicodeNewlineNode(options RegexOptions) *RegexNode { + options &^= IgnoreCase + + lead, optional := '\r', '\n' + if options&RightToLeft != 0 { + // Stored concatenations are already in execution order. In RTL, consume + // LF first and then the optional preceding CR. + lead, optional = '\n', '\r' + } + + newlines := &CharSet{} + for _, ch := range []rune{'\n', '\v', '\f', '\u0085', '\u2028', '\u2029'} { + if ch != lead { + newlines.addChar(ch) + } + } + if '\r' != lead { + newlines.addChar('\r') + } + + crlf := newRegexNode(NtConcatenate, options) + crlf.addChild(newRegexNodeCh(NtOne, options, lead)) + crlf.addChild(newRegexNodeCh(NtOne, options, optional).makeQuantifier(false, 0, 1)) + + alternate := newRegexNode(NtAlternate, options) + alternate.addChild(newRegexNodeSet(NtSet, options, newlines)) + alternate.addChild(crlf) + + atomic := newRegexNode(NtAtomic, options) + atomic.addChild(alternate) + return atomic +} + +// scanQuoted scans the literal text after \Q through the next \E, or through +// the end of the pattern when there is no terminator. +func (p *parser) scanQuoted() []rune { + start := p.textpos() + var quoted []rune + for p.charsRight() > 0 { + if p.rightChar(0) == '\\' && p.charsRight() > 1 && p.rightChar(1) == 'E' { + quoted = append(quoted, p.pattern[start:p.textpos()]...) + p.moveRight(2) + return quoted + } + p.moveRight(1) + } + return append(quoted, p.pattern[start:p.textpos()]...) +} + // Scans \-style backreferences and character escapes func (p *parser) scanBasicBackslash(scanOnly bool) (*RegexNode, error) { if p.charsRight() == 0 { @@ -1684,6 +1773,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) { inRange := false firstChar := true closed := false + var quoted []rune var cc *CharSet if !scanOnly { @@ -1697,11 +1787,33 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) { } } - for ; p.charsRight() > 0; firstChar = false { + for ; p.charsRight() > 0 || len(quoted) > 0; firstChar = false { fTranslatedChar := false - ch = p.moveRightGetChar() + for { + if len(quoted) > 0 { + ch = quoted[0] + quoted = quoted[1:] + fTranslatedChar = true + break + } + ch = p.moveRightGetChar() + if ch == '\\' && !p.useOptionE() && p.charsRight() > 0 && p.rightChar(0) == 'Q' { + p.moveRight(1) + quoted = p.scanQuoted() + if len(quoted) == 0 { + if p.charsRight() == 0 { + break + } + continue + } + continue + } + break + } if ch == ']' { - if !firstChar { + if fTranslatedChar { + // A quoted closing bracket is an ordinary class member. + } else if !firstChar { closed = true break } else if p.useOptionE() { @@ -1712,7 +1824,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) { break } - } else if ch == '\\' && p.charsRight() > 0 { + } else if ch == '\\' && !fTranslatedChar && p.charsRight() > 0 { switch ch = p.moveRightGetChar(); ch { case 'D', 'd': if !scanOnly { @@ -1815,7 +1927,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) { } fTranslatedChar = true } - } else if ch == '[' { + } else if ch == '[' && !fTranslatedChar { // This is code for Posix style properties - [:Ll:] or [:IsTibetan:]. // It currently doesn't do anything other than skip the whole thing! if p.charsRight() > 0 && p.rightChar(0) == ':' && !inRange { @@ -2291,11 +2403,54 @@ func (p *parser) addConcatenate() { } // Finish the current quantifiable (when a quantifier is found) -func (p *parser) addConcatenate3(lazy bool, min, max int) { - p.concatenation.addChild(p.unit.makeQuantifier(lazy, min, max)) +func (p *parser) addConcatenate3(lazy, possessive bool, min, max int) { + node := p.unit.makeQuantifier(lazy, min, max) + if possessive { + atomic := newRegexNode(NtAtomic, p.options) + atomic.addChild(node) + node = atomic + } + p.concatenation.addChild(node) p.unit = nil } +// addQuotedUnit adds the literal characters scanned by \Q...\E. All but the +// final character are completed immediately so a following quantifier applies +// to the final literal character, just as if each character had been escaped. +func (p *parser) addQuotedUnit(node *RegexNode) { + quoted := node.Str + if len(quoted) == 0 { + p.restoreLastUnit() + if p.unit == nil { + p.unit = newRegexNode(NtEmpty, p.options) + } + return + } + for _, ch := range quoted[:len(quoted)-1] { + p.concatenation.addChild(newRegexNodeCh(NtOne, p.options, ch)) + } + p.unit = newRegexNodeCh(NtOne, p.options, quoted[len(quoted)-1]) +} + +// restoreLastUnit makes an immediately preceding literal quantifiable again +// when an empty \Q\E appears between it and its quantifier. +func (p *parser) restoreLastUnit() { + if len(p.concatenation.Children) == 0 { + return + } + last := len(p.concatenation.Children) - 1 + node := p.concatenation.Children[last] + p.concatenation.Children = p.concatenation.Children[:last] + node.Parent = nil + if node.T == NtMulti && len(node.Str) > 1 { + prefix := append([]rune(nil), node.Str[:len(node.Str)-1]...) + p.concatenation.addChild(newRegexNodeStr(NtMulti, node.Options, prefix)) + p.unit = newRegexNodeCh(NtOne, p.options, node.Str[len(node.Str)-1]) + return + } + p.unit = node +} + // Sets the current unit to a single char node func (p *parser) addUnitOne(ch rune) { p.unit = newRegexNodeCh(NtOne, p.options, ch) diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/prefix.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/prefix.go index af5a8c06..0f6ad4aa 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/prefix.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/prefix.go @@ -245,6 +245,9 @@ func (s *regexFcd) calculateFC(nt NodeType, node *RegexNode, CurIndex int) { case NtSetloop, NtSetlazy, NtSetloopatomic: s.pushFC(regexFc{cc: node.Set.Copy(), nullable: node.M == 0, caseInsensitive: ci}) + case NtGrapheme: + s.pushFC(regexFc{cc: *AnyClass(), nullable: false}) + case NtRef: s.pushFC(regexFc{cc: *AnyClass(), nullable: true, caseInsensitive: false}) diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/prefixanalyzer.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/prefixanalyzer.go index 0490585b..4c421053 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/prefixanalyzer.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/prefixanalyzer.go @@ -123,6 +123,11 @@ func tryFindFirstCharClass(node *RegexNode, ccIn **CharSet) int { } return 0 + case NtGrapheme: + // Every rune can begin a grapheme, so there is no useful candidate + // restriction to derive. + return 0 + // Zero-width elements. These don't contribute to the starting set, so return null to indicate a caller // should keep looking past them. case NtEmpty, NtNothing, NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary, @@ -532,6 +537,9 @@ func findPrefixesCore(node *RegexNode, res *[]*bytes.Buffer, ignoreCase bool) bo // that comprise the set. For case-insensitive, we need the set to be two ASCII letters that case fold to the same thing. // As with One and loops, set loops are handled the same as sets up to the min iteration limit. case NtSet, NtSetloop, NtSetlazy, NtSetloopatomic: + if node.Set == nil || node.Set.IsNegated() { + return false + } setChars := node.Set.GetSetChars(maxPrefixes) @@ -895,10 +903,11 @@ func tryFindRawFixedSets(node *RegexNode, res *[]FixedDistanceSet, distance *int combined[fixedSet.Distance] = v } } else { + setCopy := fixedSet.Set.Copy() combined[fixedSet.Distance] = struct { Set *CharSet Count int - }{Set: fixedSet.Set, Count: 1} + }{Set: &setCopy, Count: 1} } } } @@ -1034,50 +1043,6 @@ func sumFrequencies(chars []rune) float32 { return sum } -func hasHighFrequencyChars(set FixedDistanceSet) bool { - if set.Negated { - return true - } - - // Sets without extracted chars can't be frequency-analyzed. - // Single-char sets use IndexOf, which is a strong filter regardless of frequency. - if len(set.Chars) <= 1 { - return false - } - - totalFrequency := sumFrequencies(set.Chars) - - // If the average frequency of the set's chars exceeds this threshold, the - // characters are common enough that a multi-string search may be a better filter. - const highFrequencyThreshold = 0.6 - return totalFrequency >= highFrequencyThreshold*float32(len(set.Chars)) -} - -func mayContainCaseInsensitiveMatching(node *RegexNode) bool { - if node.Options&IgnoreCase != 0 { - return true - } - - if node.Set != nil { - chars := node.Set.GetSetChars(maxPrefixes) - for _, ch := range chars { - if participatesInCaseConversion(ch) && - slices.Contains(chars, unicode.ToLower(ch)) && - slices.Contains(chars, unicode.ToUpper(ch)) { - return true - } - } - } - - for _, child := range node.Children { - if mayContainCaseInsensitiveMatching(child) { - return true - } - } - - return false -} - // Percent occurrences in source text (100 * char count / total count) var frequency = []float32{ 0.000 /* '\x00' */, 0.000 /* '\x01' */, 0.000 /* '\x02' */, 0.000 /* '\x03' */, 0.000 /* '\x04' */, 0.000 /* '\x05' */, 0.000 /* '\x06' */, 0.000, /* '\x07' */ diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/tree.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/tree.go index f4b8b6a2..bc754a2a 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/tree.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/tree.go @@ -128,6 +128,8 @@ const ( NtSetloopatomic NodeType = 45 // Updates the bumpalong position to the current position. NtUpdateBumpalong NodeType = 46 + // Matches one Unicode extended grapheme cluster (\X). + NtGrapheme NodeType = 47 ) func newRegexNode(t NodeType, opt RegexOptions) *RegexNode { @@ -693,10 +695,117 @@ func (n *RegexNode) reduceAtomic() *RegexNode { // For everything else, try to reduce ending backtracking of the last contained expression. default: child.eliminateEndingBacktracking() + if child.T == NtAlternate && child.hasDisjointStartingSets() && child.hasOnlyIntrinsicallyAtomicBranches() { + return child + } return atomic } } +// hasDisjointStartingSets reports whether every branch is non-nullable and +// begins with a character set that cannot overlap any other branch. In that +// case, once one branch has matched, no sibling branch could match the same +// starting position. +func (n *RegexNode) hasDisjointStartingSets() bool { + _, ok := n.disjointStartingSets() + return ok +} + +// disjointStartingSets returns the non-nullable first-character set for each +// branch when all of those sets are pairwise disjoint. +func (n *RegexNode) disjointStartingSets() ([]*CharSet, bool) { + sets := make([]*CharSet, 0, len(n.Children)) + for _, branch := range n.Children { + var set *CharSet + if tryFindFirstCharClass(branch, &set) != 1 || set == nil { + return nil, false + } + for _, previous := range sets { + if set.MayOverlap(previous) { + return nil, false + } + } + sets = append(sets, set) + } + return sets, true +} + +// disjointAtomicBranchSets returns the first-character sets when an +// alternation can select a branch without leaving a backtracking choice. +func (n *RegexNode) disjointAtomicBranchSets() ([]*CharSet, bool) { + if n.T != NtAlternate || !n.hasOnlyIntrinsicallyAtomicBranches() { + return nil, false + } + return n.disjointStartingSets() +} + +// dispatchCandidates returns disjoint branch sets and the leading nodes that +// can be folded into a consuming Dispatch instruction. +func (n *RegexNode) dispatchCandidates() ([]*CharSet, []*RegexNode, []bool, bool) { + if len(n.Children) >= 1<<16 { + return nil, nil, nil, false + } + sets, ok := n.disjointAtomicBranchSets() + if !ok { + return nil, nil, nil, false + } + + leaders := make([]*RegexNode, len(n.Children)) + complete := make([]bool, len(n.Children)) + for i := range leaders { + branch := n.Children[i] + leader := branch + if leader.T == NtConcatenate && len(leader.Children) > 0 { + leader = leader.Children[0] + } + switch leader.T { + case NtOne, NtNotone, NtSet, NtMulti: + leaders[i] = leader + default: + return nil, nil, nil, false + } + complete[i] = leader == branch && (leader.T != NtMulti || len(leader.Str) == 1) + } + return sets, leaders, complete, true +} + +func (n *RegexNode) hasOnlyIntrinsicallyAtomicBranches() bool { + for _, branch := range n.Children { + if !branch.isIntrinsicallyAtomic() { + return false + } + } + return true +} + +// isIntrinsicallyAtomic reports whether a successful match of this node has no +// alternative input-consuming path to explore if something later fails. +func (n *RegexNode) isIntrinsicallyAtomic() bool { + switch n.T { + case NtOne, NtNotone, NtSet, NtMulti, NtRef, NtGrapheme, + NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary, + NtBeginning, NtStart, NtEndZ, NtEnd, NtNothing, NtEmpty, NtUpdateBumpalong, + NtOneloopatomic, NtNotoneloopatomic, NtSetloopatomic, + NtPosLook, NtNegLook, NtAtomic: + return true + + case NtOneloop, NtNotoneloop, NtSetloop, NtOnelazy, NtNotonelazy, NtSetlazy: + return n.M == n.N + + case NtCapture, NtGroup: + return len(n.Children) == 1 && n.Children[0].isIntrinsicallyAtomic() + + case NtConcatenate: + for _, child := range n.Children { + if !child.isIntrinsicallyAtomic() { + return false + } + } + return true + } + return false +} + func (n *RegexNode) makeLoopAtomic() { switch n.T { @@ -1680,6 +1789,12 @@ func (n *RegexNode) reduceConcatenationWithAdjacentStrings() { // Nested repeaters just get multiplied with each other if they're not // too lumpy func (n *RegexNode) reduceRep() *RegexNode { + // Repeating a literal empty expression has no observable effect. In addition to + // being unnecessary, retaining the loop would require one backtracking frame per + // mandatory iteration even though no input is consumed. + if len(n.Children) == 1 && n.Children[0].T == NtEmpty { + return n.Children[0] + } u := n t := n.T @@ -1865,7 +1980,7 @@ func (n *RegexNode) makeQuantifier(lazy bool, min, max int) *RegexNode { // If the result is 0, there is no minimum we can enforce. func (n *RegexNode) ComputeMinLength() int { switch n.T { - case NtOne, NtNotone, NtSet: + case NtOne, NtNotone, NtSet, NtGrapheme: // single char return 1 case NtMulti: @@ -1937,6 +2052,8 @@ func (n *RegexNode) computeMaxLength() int { switch n.T { case NtOne, NtNotone, NtSet: return 1 + case NtGrapheme: + return -1 case NtMulti: return len(n.Str) case NtNotonelazy, NtNotoneloop, NtNotoneloopatomic, @@ -2049,6 +2166,7 @@ var typeStr = []string{ "ECMABoundary", "NonECMABoundary", "OneloopAtomic", "NotoneloopAtomic", "SetloopAtomic", "UpdateBumpalong", + "Grapheme", } func (n *RegexNode) Description() string { diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/unicode_alias_tables.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/unicode_alias_tables.go index a165b57e..7a231d7b 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/unicode_alias_tables.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/unicode_alias_tables.go @@ -41,6 +41,9 @@ var unicodeAliasCategories = map[string]*unicode.RangeTable{ "Grapheme_Cluster_Break=T": unicodeAliasGrapheme_Cluster_Break_T, "Grapheme_Cluster_Break=V": unicodeAliasGrapheme_Cluster_Break_V, "Grapheme_Cluster_Break=ZWJ": unicodeAliasGrapheme_Cluster_Break_ZWJ, + "Indic_Conjunct_Break=Consonant": unicodeAliasIndic_Conjunct_Break_Consonant, + "Indic_Conjunct_Break=Extend": unicodeAliasIndic_Conjunct_Break_Extend, + "Indic_Conjunct_Break=Linker": unicodeAliasIndic_Conjunct_Break_Linker, "Math": unicodeAliasMath, "Sentence_Break=ATerm": unicodeAliasSentence_Break_ATerm, "Sentence_Break=CR": unicodeAliasSentence_Break_CR, @@ -90,6 +93,8 @@ var unicodeSupportedPropertyAliases = map[string]string{ "extpict": "Extended_Pictographic", "gcb": "Grapheme_Cluster_Break", "graphemeclusterbreak": "Grapheme_Cluster_Break", + "incb": "Indic_Conjunct_Break", + "indicconjunctbreak": "Indic_Conjunct_Break", "math": "Math", "sb": "Sentence_Break", "sentencebreak": "Sentence_Break", @@ -118,6 +123,11 @@ var unicodeSupportedPropertyValueAliases = map[string]map[string]string{ "v": "V", "zwj": "ZWJ", }, + "Indic_Conjunct_Break": { + "consonant": "Consonant", + "extend": "Extend", + "linker": "Linker", + }, "Sentence_Break": { "at": "ATerm", "aterm": "ATerm", @@ -6056,3 +6066,529 @@ var unicodeAliasWord_Break_ZWJ = &unicode.RangeTable{ {Lo: 0x200D, Hi: 0x200D, Stride: 1}, }, } + +var unicodeAliasIndic_Conjunct_Break_Consonant = &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 0x915, Hi: 0x939, Stride: 1}, + {Lo: 0x958, Hi: 0x95F, Stride: 1}, + {Lo: 0x978, Hi: 0x97F, Stride: 1}, + {Lo: 0x995, Hi: 0x9A8, Stride: 1}, + {Lo: 0x9AA, Hi: 0x9B0, Stride: 1}, + {Lo: 0x9B2, Hi: 0x9B2, Stride: 1}, + {Lo: 0x9B6, Hi: 0x9B9, Stride: 1}, + {Lo: 0x9DC, Hi: 0x9DD, Stride: 1}, + {Lo: 0x9DF, Hi: 0x9DF, Stride: 1}, + {Lo: 0x9F0, Hi: 0x9F1, Stride: 1}, + {Lo: 0xA95, Hi: 0xAA8, Stride: 1}, + {Lo: 0xAAA, Hi: 0xAB0, Stride: 1}, + {Lo: 0xAB2, Hi: 0xAB3, Stride: 1}, + {Lo: 0xAB5, Hi: 0xAB9, Stride: 1}, + {Lo: 0xAF9, Hi: 0xAF9, Stride: 1}, + {Lo: 0xB15, Hi: 0xB28, Stride: 1}, + {Lo: 0xB2A, Hi: 0xB30, Stride: 1}, + {Lo: 0xB32, Hi: 0xB33, Stride: 1}, + {Lo: 0xB35, Hi: 0xB39, Stride: 1}, + {Lo: 0xB5C, Hi: 0xB5D, Stride: 1}, + {Lo: 0xB5F, Hi: 0xB5F, Stride: 1}, + {Lo: 0xB71, Hi: 0xB71, Stride: 1}, + {Lo: 0xC15, Hi: 0xC28, Stride: 1}, + {Lo: 0xC2A, Hi: 0xC39, Stride: 1}, + {Lo: 0xC58, Hi: 0xC5A, Stride: 1}, + {Lo: 0xD15, Hi: 0xD3A, Stride: 1}, + {Lo: 0x1000, Hi: 0x102A, Stride: 1}, + {Lo: 0x103F, Hi: 0x103F, Stride: 1}, + {Lo: 0x1050, Hi: 0x1055, Stride: 1}, + {Lo: 0x105A, Hi: 0x105D, Stride: 1}, + {Lo: 0x1061, Hi: 0x1061, Stride: 1}, + {Lo: 0x1065, Hi: 0x1066, Stride: 1}, + {Lo: 0x106E, Hi: 0x1070, Stride: 1}, + {Lo: 0x1075, Hi: 0x1081, Stride: 1}, + {Lo: 0x108E, Hi: 0x108E, Stride: 1}, + {Lo: 0x1780, Hi: 0x17B3, Stride: 1}, + {Lo: 0x1A20, Hi: 0x1A54, Stride: 1}, + {Lo: 0x1B0B, Hi: 0x1B0C, Stride: 1}, + {Lo: 0x1B13, Hi: 0x1B33, Stride: 1}, + {Lo: 0x1B45, Hi: 0x1B4C, Stride: 1}, + {Lo: 0x1B83, Hi: 0x1BA0, Stride: 1}, + {Lo: 0x1BAE, Hi: 0x1BAF, Stride: 1}, + {Lo: 0x1BBB, Hi: 0x1BBD, Stride: 1}, + {Lo: 0xA989, Hi: 0xA98B, Stride: 1}, + {Lo: 0xA98F, Hi: 0xA9B2, Stride: 1}, + {Lo: 0xA9E0, Hi: 0xA9E4, Stride: 1}, + {Lo: 0xA9E7, Hi: 0xA9EF, Stride: 1}, + {Lo: 0xA9FA, Hi: 0xA9FE, Stride: 1}, + {Lo: 0xAA60, Hi: 0xAA6F, Stride: 1}, + {Lo: 0xAA71, Hi: 0xAA73, Stride: 1}, + {Lo: 0xAA7A, Hi: 0xAA7A, Stride: 1}, + {Lo: 0xAA7E, Hi: 0xAA7F, Stride: 1}, + {Lo: 0xAAE0, Hi: 0xAAEA, Stride: 1}, + {Lo: 0xABC0, Hi: 0xABDA, Stride: 1}, + }, + R32: []unicode.Range32{ + {Lo: 0x10A00, Hi: 0x10A00, Stride: 1}, + {Lo: 0x10A10, Hi: 0x10A13, Stride: 1}, + {Lo: 0x10A15, Hi: 0x10A17, Stride: 1}, + {Lo: 0x10A19, Hi: 0x10A35, Stride: 1}, + {Lo: 0x11103, Hi: 0x11126, Stride: 1}, + {Lo: 0x11144, Hi: 0x11144, Stride: 1}, + {Lo: 0x11147, Hi: 0x11147, Stride: 1}, + {Lo: 0x11380, Hi: 0x11389, Stride: 1}, + {Lo: 0x1138B, Hi: 0x1138B, Stride: 1}, + {Lo: 0x1138E, Hi: 0x1138E, Stride: 1}, + {Lo: 0x11390, Hi: 0x113B5, Stride: 1}, + {Lo: 0x11900, Hi: 0x11906, Stride: 1}, + {Lo: 0x11909, Hi: 0x11909, Stride: 1}, + {Lo: 0x1190C, Hi: 0x11913, Stride: 1}, + {Lo: 0x11915, Hi: 0x11916, Stride: 1}, + {Lo: 0x11918, Hi: 0x1192F, Stride: 1}, + {Lo: 0x11A00, Hi: 0x11A00, Stride: 1}, + {Lo: 0x11A0B, Hi: 0x11A32, Stride: 1}, + {Lo: 0x11A50, Hi: 0x11A50, Stride: 1}, + {Lo: 0x11A5C, Hi: 0x11A83, Stride: 1}, + {Lo: 0x11F04, Hi: 0x11F10, Stride: 1}, + {Lo: 0x11F12, Hi: 0x11F33, Stride: 1}, + }, +} + +var unicodeAliasIndic_Conjunct_Break_Extend = &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 0x300, Hi: 0x36F, Stride: 1}, + {Lo: 0x483, Hi: 0x487, Stride: 1}, + {Lo: 0x488, Hi: 0x489, Stride: 1}, + {Lo: 0x591, Hi: 0x5BD, Stride: 1}, + {Lo: 0x5BF, Hi: 0x5BF, Stride: 1}, + {Lo: 0x5C1, Hi: 0x5C2, Stride: 1}, + {Lo: 0x5C4, Hi: 0x5C5, Stride: 1}, + {Lo: 0x5C7, Hi: 0x5C7, Stride: 1}, + {Lo: 0x610, Hi: 0x61A, Stride: 1}, + {Lo: 0x64B, Hi: 0x65F, Stride: 1}, + {Lo: 0x670, Hi: 0x670, Stride: 1}, + {Lo: 0x6D6, Hi: 0x6DC, Stride: 1}, + {Lo: 0x6DF, Hi: 0x6E4, Stride: 1}, + {Lo: 0x6E7, Hi: 0x6E8, Stride: 1}, + {Lo: 0x6EA, Hi: 0x6ED, Stride: 1}, + {Lo: 0x711, Hi: 0x711, Stride: 1}, + {Lo: 0x730, Hi: 0x74A, Stride: 1}, + {Lo: 0x7A6, Hi: 0x7B0, Stride: 1}, + {Lo: 0x7EB, Hi: 0x7F3, Stride: 1}, + {Lo: 0x7FD, Hi: 0x7FD, Stride: 1}, + {Lo: 0x816, Hi: 0x819, Stride: 1}, + {Lo: 0x81B, Hi: 0x823, Stride: 1}, + {Lo: 0x825, Hi: 0x827, Stride: 1}, + {Lo: 0x829, Hi: 0x82D, Stride: 1}, + {Lo: 0x859, Hi: 0x85B, Stride: 1}, + {Lo: 0x897, Hi: 0x89F, Stride: 1}, + {Lo: 0x8CA, Hi: 0x8E1, Stride: 1}, + {Lo: 0x8E3, Hi: 0x902, Stride: 1}, + {Lo: 0x93A, Hi: 0x93A, Stride: 1}, + {Lo: 0x93C, Hi: 0x93C, Stride: 1}, + {Lo: 0x941, Hi: 0x948, Stride: 1}, + {Lo: 0x951, Hi: 0x957, Stride: 1}, + {Lo: 0x962, Hi: 0x963, Stride: 1}, + {Lo: 0x981, Hi: 0x981, Stride: 1}, + {Lo: 0x9BC, Hi: 0x9BC, Stride: 1}, + {Lo: 0x9BE, Hi: 0x9BE, Stride: 1}, + {Lo: 0x9C1, Hi: 0x9C4, Stride: 1}, + {Lo: 0x9D7, Hi: 0x9D7, Stride: 1}, + {Lo: 0x9E2, Hi: 0x9E3, Stride: 1}, + {Lo: 0x9FE, Hi: 0x9FE, Stride: 1}, + {Lo: 0xA01, Hi: 0xA02, Stride: 1}, + {Lo: 0xA3C, Hi: 0xA3C, Stride: 1}, + {Lo: 0xA41, Hi: 0xA42, Stride: 1}, + {Lo: 0xA47, Hi: 0xA48, Stride: 1}, + {Lo: 0xA4B, Hi: 0xA4D, Stride: 1}, + {Lo: 0xA51, Hi: 0xA51, Stride: 1}, + {Lo: 0xA70, Hi: 0xA71, Stride: 1}, + {Lo: 0xA75, Hi: 0xA75, Stride: 1}, + {Lo: 0xA81, Hi: 0xA82, Stride: 1}, + {Lo: 0xABC, Hi: 0xABC, Stride: 1}, + {Lo: 0xAC1, Hi: 0xAC5, Stride: 1}, + {Lo: 0xAC7, Hi: 0xAC8, Stride: 1}, + {Lo: 0xAE2, Hi: 0xAE3, Stride: 1}, + {Lo: 0xAFA, Hi: 0xAFF, Stride: 1}, + {Lo: 0xB01, Hi: 0xB01, Stride: 1}, + {Lo: 0xB3C, Hi: 0xB3C, Stride: 1}, + {Lo: 0xB3E, Hi: 0xB3E, Stride: 1}, + {Lo: 0xB3F, Hi: 0xB3F, Stride: 1}, + {Lo: 0xB41, Hi: 0xB44, Stride: 1}, + {Lo: 0xB55, Hi: 0xB56, Stride: 1}, + {Lo: 0xB57, Hi: 0xB57, Stride: 1}, + {Lo: 0xB62, Hi: 0xB63, Stride: 1}, + {Lo: 0xB82, Hi: 0xB82, Stride: 1}, + {Lo: 0xBBE, Hi: 0xBBE, Stride: 1}, + {Lo: 0xBC0, Hi: 0xBC0, Stride: 1}, + {Lo: 0xBCD, Hi: 0xBCD, Stride: 1}, + {Lo: 0xBD7, Hi: 0xBD7, Stride: 1}, + {Lo: 0xC00, Hi: 0xC00, Stride: 1}, + {Lo: 0xC04, Hi: 0xC04, Stride: 1}, + {Lo: 0xC3C, Hi: 0xC3C, Stride: 1}, + {Lo: 0xC3E, Hi: 0xC40, Stride: 1}, + {Lo: 0xC46, Hi: 0xC48, Stride: 1}, + {Lo: 0xC4A, Hi: 0xC4C, Stride: 1}, + {Lo: 0xC55, Hi: 0xC56, Stride: 1}, + {Lo: 0xC62, Hi: 0xC63, Stride: 1}, + {Lo: 0xC81, Hi: 0xC81, Stride: 1}, + {Lo: 0xCBC, Hi: 0xCBC, Stride: 1}, + {Lo: 0xCBF, Hi: 0xCBF, Stride: 1}, + {Lo: 0xCC0, Hi: 0xCC0, Stride: 1}, + {Lo: 0xCC2, Hi: 0xCC2, Stride: 1}, + {Lo: 0xCC6, Hi: 0xCC6, Stride: 1}, + {Lo: 0xCC7, Hi: 0xCC8, Stride: 1}, + {Lo: 0xCCA, Hi: 0xCCB, Stride: 1}, + {Lo: 0xCCC, Hi: 0xCCD, Stride: 1}, + {Lo: 0xCD5, Hi: 0xCD6, Stride: 1}, + {Lo: 0xCE2, Hi: 0xCE3, Stride: 1}, + {Lo: 0xD00, Hi: 0xD01, Stride: 1}, + {Lo: 0xD3B, Hi: 0xD3C, Stride: 1}, + {Lo: 0xD3E, Hi: 0xD3E, Stride: 1}, + {Lo: 0xD41, Hi: 0xD44, Stride: 1}, + {Lo: 0xD57, Hi: 0xD57, Stride: 1}, + {Lo: 0xD62, Hi: 0xD63, Stride: 1}, + {Lo: 0xD81, Hi: 0xD81, Stride: 1}, + {Lo: 0xDCA, Hi: 0xDCA, Stride: 1}, + {Lo: 0xDCF, Hi: 0xDCF, Stride: 1}, + {Lo: 0xDD2, Hi: 0xDD4, Stride: 1}, + {Lo: 0xDD6, Hi: 0xDD6, Stride: 1}, + {Lo: 0xDDF, Hi: 0xDDF, Stride: 1}, + {Lo: 0xE31, Hi: 0xE31, Stride: 1}, + {Lo: 0xE34, Hi: 0xE3A, Stride: 1}, + {Lo: 0xE47, Hi: 0xE4E, Stride: 1}, + {Lo: 0xEB1, Hi: 0xEB1, Stride: 1}, + {Lo: 0xEB4, Hi: 0xEBC, Stride: 1}, + {Lo: 0xEC8, Hi: 0xECE, Stride: 1}, + {Lo: 0xF18, Hi: 0xF19, Stride: 1}, + {Lo: 0xF35, Hi: 0xF35, Stride: 1}, + {Lo: 0xF37, Hi: 0xF37, Stride: 1}, + {Lo: 0xF39, Hi: 0xF39, Stride: 1}, + {Lo: 0xF71, Hi: 0xF7E, Stride: 1}, + {Lo: 0xF80, Hi: 0xF84, Stride: 1}, + {Lo: 0xF86, Hi: 0xF87, Stride: 1}, + {Lo: 0xF8D, Hi: 0xF97, Stride: 1}, + {Lo: 0xF99, Hi: 0xFBC, Stride: 1}, + {Lo: 0xFC6, Hi: 0xFC6, Stride: 1}, + {Lo: 0x102D, Hi: 0x1030, Stride: 1}, + {Lo: 0x1032, Hi: 0x1037, Stride: 1}, + {Lo: 0x103A, Hi: 0x103A, Stride: 1}, + {Lo: 0x103D, Hi: 0x103E, Stride: 1}, + {Lo: 0x1058, Hi: 0x1059, Stride: 1}, + {Lo: 0x105E, Hi: 0x1060, Stride: 1}, + {Lo: 0x1071, Hi: 0x1074, Stride: 1}, + {Lo: 0x1082, Hi: 0x1082, Stride: 1}, + {Lo: 0x1085, Hi: 0x1086, Stride: 1}, + {Lo: 0x108D, Hi: 0x108D, Stride: 1}, + {Lo: 0x109D, Hi: 0x109D, Stride: 1}, + {Lo: 0x135D, Hi: 0x135F, Stride: 1}, + {Lo: 0x1712, Hi: 0x1714, Stride: 1}, + {Lo: 0x1715, Hi: 0x1715, Stride: 1}, + {Lo: 0x1732, Hi: 0x1733, Stride: 1}, + {Lo: 0x1734, Hi: 0x1734, Stride: 1}, + {Lo: 0x1752, Hi: 0x1753, Stride: 1}, + {Lo: 0x1772, Hi: 0x1773, Stride: 1}, + {Lo: 0x17B4, Hi: 0x17B5, Stride: 1}, + {Lo: 0x17B7, Hi: 0x17BD, Stride: 1}, + {Lo: 0x17C6, Hi: 0x17C6, Stride: 1}, + {Lo: 0x17C9, Hi: 0x17D1, Stride: 1}, + {Lo: 0x17D3, Hi: 0x17D3, Stride: 1}, + {Lo: 0x17DD, Hi: 0x17DD, Stride: 1}, + {Lo: 0x180B, Hi: 0x180D, Stride: 1}, + {Lo: 0x180F, Hi: 0x180F, Stride: 1}, + {Lo: 0x1885, Hi: 0x1886, Stride: 1}, + {Lo: 0x18A9, Hi: 0x18A9, Stride: 1}, + {Lo: 0x1920, Hi: 0x1922, Stride: 1}, + {Lo: 0x1927, Hi: 0x1928, Stride: 1}, + {Lo: 0x1932, Hi: 0x1932, Stride: 1}, + {Lo: 0x1939, Hi: 0x193B, Stride: 1}, + {Lo: 0x1A17, Hi: 0x1A18, Stride: 1}, + {Lo: 0x1A1B, Hi: 0x1A1B, Stride: 1}, + {Lo: 0x1A56, Hi: 0x1A56, Stride: 1}, + {Lo: 0x1A58, Hi: 0x1A5E, Stride: 1}, + {Lo: 0x1A62, Hi: 0x1A62, Stride: 1}, + {Lo: 0x1A65, Hi: 0x1A6C, Stride: 1}, + {Lo: 0x1A73, Hi: 0x1A7C, Stride: 1}, + {Lo: 0x1A7F, Hi: 0x1A7F, Stride: 1}, + {Lo: 0x1AB0, Hi: 0x1ABD, Stride: 1}, + {Lo: 0x1ABE, Hi: 0x1ABE, Stride: 1}, + {Lo: 0x1ABF, Hi: 0x1ADD, Stride: 1}, + {Lo: 0x1AE0, Hi: 0x1AEB, Stride: 1}, + {Lo: 0x1B00, Hi: 0x1B03, Stride: 1}, + {Lo: 0x1B34, Hi: 0x1B34, Stride: 1}, + {Lo: 0x1B35, Hi: 0x1B35, Stride: 1}, + {Lo: 0x1B36, Hi: 0x1B3A, Stride: 1}, + {Lo: 0x1B3B, Hi: 0x1B3B, Stride: 1}, + {Lo: 0x1B3C, Hi: 0x1B3C, Stride: 1}, + {Lo: 0x1B3D, Hi: 0x1B3D, Stride: 1}, + {Lo: 0x1B42, Hi: 0x1B42, Stride: 1}, + {Lo: 0x1B43, Hi: 0x1B43, Stride: 1}, + {Lo: 0x1B6B, Hi: 0x1B73, Stride: 1}, + {Lo: 0x1B80, Hi: 0x1B81, Stride: 1}, + {Lo: 0x1BA2, Hi: 0x1BA5, Stride: 1}, + {Lo: 0x1BA8, Hi: 0x1BA9, Stride: 1}, + {Lo: 0x1BAA, Hi: 0x1BAA, Stride: 1}, + {Lo: 0x1BAC, Hi: 0x1BAD, Stride: 1}, + {Lo: 0x1BE6, Hi: 0x1BE6, Stride: 1}, + {Lo: 0x1BE8, Hi: 0x1BE9, Stride: 1}, + {Lo: 0x1BED, Hi: 0x1BED, Stride: 1}, + {Lo: 0x1BEF, Hi: 0x1BF1, Stride: 1}, + {Lo: 0x1BF2, Hi: 0x1BF3, Stride: 1}, + {Lo: 0x1C2C, Hi: 0x1C33, Stride: 1}, + {Lo: 0x1C36, Hi: 0x1C37, Stride: 1}, + {Lo: 0x1CD0, Hi: 0x1CD2, Stride: 1}, + {Lo: 0x1CD4, Hi: 0x1CE0, Stride: 1}, + {Lo: 0x1CE2, Hi: 0x1CE8, Stride: 1}, + {Lo: 0x1CED, Hi: 0x1CED, Stride: 1}, + {Lo: 0x1CF4, Hi: 0x1CF4, Stride: 1}, + {Lo: 0x1CF8, Hi: 0x1CF9, Stride: 1}, + {Lo: 0x1DC0, Hi: 0x1DFF, Stride: 1}, + {Lo: 0x200D, Hi: 0x200D, Stride: 1}, + {Lo: 0x20D0, Hi: 0x20DC, Stride: 1}, + {Lo: 0x20DD, Hi: 0x20E0, Stride: 1}, + {Lo: 0x20E1, Hi: 0x20E1, Stride: 1}, + {Lo: 0x20E2, Hi: 0x20E4, Stride: 1}, + {Lo: 0x20E5, Hi: 0x20F0, Stride: 1}, + {Lo: 0x2CEF, Hi: 0x2CF1, Stride: 1}, + {Lo: 0x2D7F, Hi: 0x2D7F, Stride: 1}, + {Lo: 0x2DE0, Hi: 0x2DFF, Stride: 1}, + {Lo: 0x302A, Hi: 0x302D, Stride: 1}, + {Lo: 0x302E, Hi: 0x302F, Stride: 1}, + {Lo: 0x3099, Hi: 0x309A, Stride: 1}, + {Lo: 0xA66F, Hi: 0xA66F, Stride: 1}, + {Lo: 0xA670, Hi: 0xA672, Stride: 1}, + {Lo: 0xA674, Hi: 0xA67D, Stride: 1}, + {Lo: 0xA69E, Hi: 0xA69F, Stride: 1}, + {Lo: 0xA6F0, Hi: 0xA6F1, Stride: 1}, + {Lo: 0xA802, Hi: 0xA802, Stride: 1}, + {Lo: 0xA806, Hi: 0xA806, Stride: 1}, + {Lo: 0xA80B, Hi: 0xA80B, Stride: 1}, + {Lo: 0xA825, Hi: 0xA826, Stride: 1}, + {Lo: 0xA82C, Hi: 0xA82C, Stride: 1}, + {Lo: 0xA8C4, Hi: 0xA8C5, Stride: 1}, + {Lo: 0xA8E0, Hi: 0xA8F1, Stride: 1}, + {Lo: 0xA8FF, Hi: 0xA8FF, Stride: 1}, + {Lo: 0xA926, Hi: 0xA92D, Stride: 1}, + {Lo: 0xA947, Hi: 0xA951, Stride: 1}, + {Lo: 0xA953, Hi: 0xA953, Stride: 1}, + {Lo: 0xA980, Hi: 0xA982, Stride: 1}, + {Lo: 0xA9B3, Hi: 0xA9B3, Stride: 1}, + {Lo: 0xA9B6, Hi: 0xA9B9, Stride: 1}, + {Lo: 0xA9BC, Hi: 0xA9BD, Stride: 1}, + {Lo: 0xA9E5, Hi: 0xA9E5, Stride: 1}, + {Lo: 0xAA29, Hi: 0xAA2E, Stride: 1}, + {Lo: 0xAA31, Hi: 0xAA32, Stride: 1}, + {Lo: 0xAA35, Hi: 0xAA36, Stride: 1}, + {Lo: 0xAA43, Hi: 0xAA43, Stride: 1}, + {Lo: 0xAA4C, Hi: 0xAA4C, Stride: 1}, + {Lo: 0xAA7C, Hi: 0xAA7C, Stride: 1}, + {Lo: 0xAAB0, Hi: 0xAAB0, Stride: 1}, + {Lo: 0xAAB2, Hi: 0xAAB4, Stride: 1}, + {Lo: 0xAAB7, Hi: 0xAAB8, Stride: 1}, + {Lo: 0xAABE, Hi: 0xAABF, Stride: 1}, + {Lo: 0xAAC1, Hi: 0xAAC1, Stride: 1}, + {Lo: 0xAAEC, Hi: 0xAAED, Stride: 1}, + {Lo: 0xABE5, Hi: 0xABE5, Stride: 1}, + {Lo: 0xABE8, Hi: 0xABE8, Stride: 1}, + {Lo: 0xABED, Hi: 0xABED, Stride: 1}, + {Lo: 0xFB1E, Hi: 0xFB1E, Stride: 1}, + {Lo: 0xFE00, Hi: 0xFE0F, Stride: 1}, + {Lo: 0xFE20, Hi: 0xFE2F, Stride: 1}, + {Lo: 0xFF9E, Hi: 0xFF9F, Stride: 1}, + }, + R32: []unicode.Range32{ + {Lo: 0x101FD, Hi: 0x101FD, Stride: 1}, + {Lo: 0x102E0, Hi: 0x102E0, Stride: 1}, + {Lo: 0x10376, Hi: 0x1037A, Stride: 1}, + {Lo: 0x10A01, Hi: 0x10A03, Stride: 1}, + {Lo: 0x10A05, Hi: 0x10A06, Stride: 1}, + {Lo: 0x10A0C, Hi: 0x10A0F, Stride: 1}, + {Lo: 0x10A38, Hi: 0x10A3A, Stride: 1}, + {Lo: 0x10AE5, Hi: 0x10AE6, Stride: 1}, + {Lo: 0x10D24, Hi: 0x10D27, Stride: 1}, + {Lo: 0x10D69, Hi: 0x10D6D, Stride: 1}, + {Lo: 0x10EAB, Hi: 0x10EAC, Stride: 1}, + {Lo: 0x10EFA, Hi: 0x10EFF, Stride: 1}, + {Lo: 0x10F46, Hi: 0x10F50, Stride: 1}, + {Lo: 0x10F82, Hi: 0x10F85, Stride: 1}, + {Lo: 0x11001, Hi: 0x11001, Stride: 1}, + {Lo: 0x11038, Hi: 0x11046, Stride: 1}, + {Lo: 0x11070, Hi: 0x11070, Stride: 1}, + {Lo: 0x11073, Hi: 0x11074, Stride: 1}, + {Lo: 0x1107F, Hi: 0x11081, Stride: 1}, + {Lo: 0x110B3, Hi: 0x110B6, Stride: 1}, + {Lo: 0x110B9, Hi: 0x110BA, Stride: 1}, + {Lo: 0x110C2, Hi: 0x110C2, Stride: 1}, + {Lo: 0x11100, Hi: 0x11102, Stride: 1}, + {Lo: 0x11127, Hi: 0x1112B, Stride: 1}, + {Lo: 0x1112D, Hi: 0x11132, Stride: 1}, + {Lo: 0x11134, Hi: 0x11134, Stride: 1}, + {Lo: 0x11173, Hi: 0x11173, Stride: 1}, + {Lo: 0x11180, Hi: 0x11181, Stride: 1}, + {Lo: 0x111B6, Hi: 0x111BE, Stride: 1}, + {Lo: 0x111C0, Hi: 0x111C0, Stride: 1}, + {Lo: 0x111C9, Hi: 0x111CC, Stride: 1}, + {Lo: 0x111CF, Hi: 0x111CF, Stride: 1}, + {Lo: 0x1122F, Hi: 0x11231, Stride: 1}, + {Lo: 0x11234, Hi: 0x11234, Stride: 1}, + {Lo: 0x11235, Hi: 0x11235, Stride: 1}, + {Lo: 0x11236, Hi: 0x11237, Stride: 1}, + {Lo: 0x1123E, Hi: 0x1123E, Stride: 1}, + {Lo: 0x11241, Hi: 0x11241, Stride: 1}, + {Lo: 0x112DF, Hi: 0x112DF, Stride: 1}, + {Lo: 0x112E3, Hi: 0x112EA, Stride: 1}, + {Lo: 0x11300, Hi: 0x11301, Stride: 1}, + {Lo: 0x1133B, Hi: 0x1133C, Stride: 1}, + {Lo: 0x1133E, Hi: 0x1133E, Stride: 1}, + {Lo: 0x11340, Hi: 0x11340, Stride: 1}, + {Lo: 0x1134D, Hi: 0x1134D, Stride: 1}, + {Lo: 0x11357, Hi: 0x11357, Stride: 1}, + {Lo: 0x11366, Hi: 0x1136C, Stride: 1}, + {Lo: 0x11370, Hi: 0x11374, Stride: 1}, + {Lo: 0x113B8, Hi: 0x113B8, Stride: 1}, + {Lo: 0x113BB, Hi: 0x113C0, Stride: 1}, + {Lo: 0x113C2, Hi: 0x113C2, Stride: 1}, + {Lo: 0x113C5, Hi: 0x113C5, Stride: 1}, + {Lo: 0x113C7, Hi: 0x113C9, Stride: 1}, + {Lo: 0x113CE, Hi: 0x113CE, Stride: 1}, + {Lo: 0x113CF, Hi: 0x113CF, Stride: 1}, + {Lo: 0x113D2, Hi: 0x113D2, Stride: 1}, + {Lo: 0x113E1, Hi: 0x113E2, Stride: 1}, + {Lo: 0x11438, Hi: 0x1143F, Stride: 1}, + {Lo: 0x11442, Hi: 0x11444, Stride: 1}, + {Lo: 0x11446, Hi: 0x11446, Stride: 1}, + {Lo: 0x1145E, Hi: 0x1145E, Stride: 1}, + {Lo: 0x114B0, Hi: 0x114B0, Stride: 1}, + {Lo: 0x114B3, Hi: 0x114B8, Stride: 1}, + {Lo: 0x114BA, Hi: 0x114BA, Stride: 1}, + {Lo: 0x114BD, Hi: 0x114BD, Stride: 1}, + {Lo: 0x114BF, Hi: 0x114C0, Stride: 1}, + {Lo: 0x114C2, Hi: 0x114C3, Stride: 1}, + {Lo: 0x115AF, Hi: 0x115AF, Stride: 1}, + {Lo: 0x115B2, Hi: 0x115B5, Stride: 1}, + {Lo: 0x115BC, Hi: 0x115BD, Stride: 1}, + {Lo: 0x115BF, Hi: 0x115C0, Stride: 1}, + {Lo: 0x115DC, Hi: 0x115DD, Stride: 1}, + {Lo: 0x11633, Hi: 0x1163A, Stride: 1}, + {Lo: 0x1163D, Hi: 0x1163D, Stride: 1}, + {Lo: 0x1163F, Hi: 0x11640, Stride: 1}, + {Lo: 0x116AB, Hi: 0x116AB, Stride: 1}, + {Lo: 0x116AD, Hi: 0x116AD, Stride: 1}, + {Lo: 0x116B0, Hi: 0x116B5, Stride: 1}, + {Lo: 0x116B6, Hi: 0x116B6, Stride: 1}, + {Lo: 0x116B7, Hi: 0x116B7, Stride: 1}, + {Lo: 0x1171D, Hi: 0x1171D, Stride: 1}, + {Lo: 0x1171F, Hi: 0x1171F, Stride: 1}, + {Lo: 0x11722, Hi: 0x11725, Stride: 1}, + {Lo: 0x11727, Hi: 0x1172B, Stride: 1}, + {Lo: 0x1182F, Hi: 0x11837, Stride: 1}, + {Lo: 0x11839, Hi: 0x1183A, Stride: 1}, + {Lo: 0x11930, Hi: 0x11930, Stride: 1}, + {Lo: 0x1193B, Hi: 0x1193C, Stride: 1}, + {Lo: 0x1193D, Hi: 0x1193D, Stride: 1}, + {Lo: 0x11943, Hi: 0x11943, Stride: 1}, + {Lo: 0x119D4, Hi: 0x119D7, Stride: 1}, + {Lo: 0x119DA, Hi: 0x119DB, Stride: 1}, + {Lo: 0x119E0, Hi: 0x119E0, Stride: 1}, + {Lo: 0x11A01, Hi: 0x11A0A, Stride: 1}, + {Lo: 0x11A33, Hi: 0x11A38, Stride: 1}, + {Lo: 0x11A3B, Hi: 0x11A3E, Stride: 1}, + {Lo: 0x11A51, Hi: 0x11A56, Stride: 1}, + {Lo: 0x11A59, Hi: 0x11A5B, Stride: 1}, + {Lo: 0x11A8A, Hi: 0x11A96, Stride: 1}, + {Lo: 0x11A98, Hi: 0x11A98, Stride: 1}, + {Lo: 0x11B60, Hi: 0x11B60, Stride: 1}, + {Lo: 0x11B62, Hi: 0x11B64, Stride: 1}, + {Lo: 0x11B66, Hi: 0x11B66, Stride: 1}, + {Lo: 0x11C30, Hi: 0x11C36, Stride: 1}, + {Lo: 0x11C38, Hi: 0x11C3D, Stride: 1}, + {Lo: 0x11C3F, Hi: 0x11C3F, Stride: 1}, + {Lo: 0x11C92, Hi: 0x11CA7, Stride: 1}, + {Lo: 0x11CAA, Hi: 0x11CB0, Stride: 1}, + {Lo: 0x11CB2, Hi: 0x11CB3, Stride: 1}, + {Lo: 0x11CB5, Hi: 0x11CB6, Stride: 1}, + {Lo: 0x11D31, Hi: 0x11D36, Stride: 1}, + {Lo: 0x11D3A, Hi: 0x11D3A, Stride: 1}, + {Lo: 0x11D3C, Hi: 0x11D3D, Stride: 1}, + {Lo: 0x11D3F, Hi: 0x11D45, Stride: 1}, + {Lo: 0x11D47, Hi: 0x11D47, Stride: 1}, + {Lo: 0x11D90, Hi: 0x11D91, Stride: 1}, + {Lo: 0x11D95, Hi: 0x11D95, Stride: 1}, + {Lo: 0x11D97, Hi: 0x11D97, Stride: 1}, + {Lo: 0x11EF3, Hi: 0x11EF4, Stride: 1}, + {Lo: 0x11F00, Hi: 0x11F01, Stride: 1}, + {Lo: 0x11F36, Hi: 0x11F3A, Stride: 1}, + {Lo: 0x11F40, Hi: 0x11F40, Stride: 1}, + {Lo: 0x11F41, Hi: 0x11F41, Stride: 1}, + {Lo: 0x11F5A, Hi: 0x11F5A, Stride: 1}, + {Lo: 0x13440, Hi: 0x13440, Stride: 1}, + {Lo: 0x13447, Hi: 0x13455, Stride: 1}, + {Lo: 0x1611E, Hi: 0x16129, Stride: 1}, + {Lo: 0x1612D, Hi: 0x1612F, Stride: 1}, + {Lo: 0x16AF0, Hi: 0x16AF4, Stride: 1}, + {Lo: 0x16B30, Hi: 0x16B36, Stride: 1}, + {Lo: 0x16F4F, Hi: 0x16F4F, Stride: 1}, + {Lo: 0x16F8F, Hi: 0x16F92, Stride: 1}, + {Lo: 0x16FE4, Hi: 0x16FE4, Stride: 1}, + {Lo: 0x16FF0, Hi: 0x16FF1, Stride: 1}, + {Lo: 0x1BC9D, Hi: 0x1BC9E, Stride: 1}, + {Lo: 0x1CF00, Hi: 0x1CF2D, Stride: 1}, + {Lo: 0x1CF30, Hi: 0x1CF46, Stride: 1}, + {Lo: 0x1D165, Hi: 0x1D166, Stride: 1}, + {Lo: 0x1D167, Hi: 0x1D169, Stride: 1}, + {Lo: 0x1D16D, Hi: 0x1D172, Stride: 1}, + {Lo: 0x1D17B, Hi: 0x1D182, Stride: 1}, + {Lo: 0x1D185, Hi: 0x1D18B, Stride: 1}, + {Lo: 0x1D1AA, Hi: 0x1D1AD, Stride: 1}, + {Lo: 0x1D242, Hi: 0x1D244, Stride: 1}, + {Lo: 0x1DA00, Hi: 0x1DA36, Stride: 1}, + {Lo: 0x1DA3B, Hi: 0x1DA6C, Stride: 1}, + {Lo: 0x1DA75, Hi: 0x1DA75, Stride: 1}, + {Lo: 0x1DA84, Hi: 0x1DA84, Stride: 1}, + {Lo: 0x1DA9B, Hi: 0x1DA9F, Stride: 1}, + {Lo: 0x1DAA1, Hi: 0x1DAAF, Stride: 1}, + {Lo: 0x1E000, Hi: 0x1E006, Stride: 1}, + {Lo: 0x1E008, Hi: 0x1E018, Stride: 1}, + {Lo: 0x1E01B, Hi: 0x1E021, Stride: 1}, + {Lo: 0x1E023, Hi: 0x1E024, Stride: 1}, + {Lo: 0x1E026, Hi: 0x1E02A, Stride: 1}, + {Lo: 0x1E08F, Hi: 0x1E08F, Stride: 1}, + {Lo: 0x1E130, Hi: 0x1E136, Stride: 1}, + {Lo: 0x1E2AE, Hi: 0x1E2AE, Stride: 1}, + {Lo: 0x1E2EC, Hi: 0x1E2EF, Stride: 1}, + {Lo: 0x1E4EC, Hi: 0x1E4EF, Stride: 1}, + {Lo: 0x1E5EE, Hi: 0x1E5EF, Stride: 1}, + {Lo: 0x1E6E3, Hi: 0x1E6E3, Stride: 1}, + {Lo: 0x1E6E6, Hi: 0x1E6E6, Stride: 1}, + {Lo: 0x1E6EE, Hi: 0x1E6EF, Stride: 1}, + {Lo: 0x1E6F5, Hi: 0x1E6F5, Stride: 1}, + {Lo: 0x1E8D0, Hi: 0x1E8D6, Stride: 1}, + {Lo: 0x1E944, Hi: 0x1E94A, Stride: 1}, + {Lo: 0x1F3FB, Hi: 0x1F3FF, Stride: 1}, + {Lo: 0xE0020, Hi: 0xE007F, Stride: 1}, + {Lo: 0xE0100, Hi: 0xE01EF, Stride: 1}, + }, +} + +var unicodeAliasIndic_Conjunct_Break_Linker = &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 0x94D, Hi: 0x94D, Stride: 1}, + {Lo: 0x9CD, Hi: 0x9CD, Stride: 1}, + {Lo: 0xACD, Hi: 0xACD, Stride: 1}, + {Lo: 0xB4D, Hi: 0xB4D, Stride: 1}, + {Lo: 0xC4D, Hi: 0xC4D, Stride: 1}, + {Lo: 0xD4D, Hi: 0xD4D, Stride: 1}, + {Lo: 0x1039, Hi: 0x1039, Stride: 1}, + {Lo: 0x17D2, Hi: 0x17D2, Stride: 1}, + {Lo: 0x1A60, Hi: 0x1A60, Stride: 1}, + {Lo: 0x1B44, Hi: 0x1B44, Stride: 1}, + {Lo: 0x1BAB, Hi: 0x1BAB, Stride: 1}, + {Lo: 0xA9C0, Hi: 0xA9C0, Stride: 1}, + {Lo: 0xAAF6, Hi: 0xAAF6, Stride: 1}, + }, + R32: []unicode.Range32{ + {Lo: 0x10A3F, Hi: 0x10A3F, Stride: 1}, + {Lo: 0x11133, Hi: 0x11133, Stride: 1}, + {Lo: 0x113D0, Hi: 0x113D0, Stride: 1}, + {Lo: 0x1193E, Hi: 0x1193E, Stride: 1}, + {Lo: 0x11A47, Hi: 0x11A47, Stride: 1}, + {Lo: 0x11A99, Hi: 0x11A99, Stride: 1}, + {Lo: 0x11F42, Hi: 0x11F42, Stride: 1}, + }, +} diff --git a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/writer.go b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/writer.go index 818b4b56..ec1818e7 100644 --- a/backend/vendor/github.com/dlclark/regexp2/v2/syntax/writer.go +++ b/backend/vendor/github.com/dlclark/regexp2/v2/syntax/writer.go @@ -4,34 +4,93 @@ import ( "bytes" "fmt" "math" + "slices" ) func Write(tree *RegexTree) (*Code, error) { - w := writer{ - intStack: make([]int, 0, 32), - emitted: make([]int, 2), - stringhash: make(map[string]int), - sethash: make(map[string]int), + w := newWriter(nil) + code, err := w.codeFromTree(tree) + if err != nil { + return nil, err } - code, err := w.codeFromTree(tree) + if slices.Contains(code.CaptureSlotInUse, false) { + quickWriter := newWriter(code.CaptureSlotInUse) + quickCode, err := quickWriter.codeFromTree(tree) + if err != nil { + return nil, err + } + if !dispatchTablesEqual(code.Dispatches, quickCode.Dispatches) { + return nil, fmt.Errorf("full and quick dispatch tables differ") + } + for i := range quickCode.Dispatches { + quickCode.Dispatches[i].Sets = code.Dispatches[i].Sets + quickCode.Dispatches[i].ASCII = code.Dispatches[i].ASCII + quickCode.Dispatches[i].ASCIIMasks = code.Dispatches[i].ASCIIMasks + } + code.QuickCodes = quickCode.Codes + code.QuickDispatches = quickCode.Dispatches + } + return code, nil +} - return code, err +func newWriter(quickCaptureSlots []bool) writer { + return writer{ + intStack: make([]int, 0, 32), + emitted: make([]int, 2), + stringhash: make(map[string]int), + sethash: make(map[string]int), + preconsumed: make(map[*RegexNode]bool), + dispatchInfo: make(map[*RegexNode]dispatchInfo), + dispatchCodes: make(map[*RegexNode]int), + dispatchGotos: make(map[*RegexNode][]int), + quickCaptureSlots: quickCaptureSlots, + } } type writer struct { emitted []int - intStack []int - curpos int - stringhash map[string]int - stringtable [][]rune - sethash map[string]int - settable []*CharSet - counting bool - count int - trackcount int - caps map[int]int + intStack []int + curpos int + stringhash map[string]int + stringtable [][]rune + sethash map[string]int + settable []*CharSet + dispatchtable []DispatchTable + preconsumed map[*RegexNode]bool + dispatchInfo map[*RegexNode]dispatchInfo + dispatchCodes map[*RegexNode]int + dispatchGotos map[*RegexNode][]int + counting bool + count int + trackcount int + caps map[int]int + quickCaptureSlots []bool +} + +type dispatchInfo struct { + sets []*CharSet + leaders []*RegexNode + complete []bool + ok bool +} + +func dispatchTablesEqual(left, right []DispatchTable) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if !slices.Equal(left[i].Sets, right[i].Sets) || + !slices.Equal(left[i].ASCIIMasks, right[i].ASCIIMasks) || + (left[i].ASCII == nil) != (right[i].ASCII == nil) { + return false + } + if left[i].ASCII != nil && *left[i].ASCII != *right[i].ASCII { + return false + } + } + return true } const ( @@ -74,6 +133,8 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) { for { if !w.counting { w.emitted = make([]int, w.count) + w.dispatchCodes = make(map[*RegexNode]int) + w.dispatchGotos = make(map[*RegexNode][]int) } curNode = tree.Root @@ -120,6 +181,9 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) { w.counting = false } + if w.quickCaptureSlots != nil { + return &Code{Codes: w.emitted, Dispatches: w.dispatchtable, TrackCount: w.trackcount}, nil + } fcPrefix := getFirstCharsPrefix(tree) prefix := getPrefix(tree) @@ -141,17 +205,57 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) { Codes: w.emitted, Strings: w.stringtable, Sets: w.settable, + Dispatches: w.dispatchtable, TrackCount: w.trackcount, Caps: w.caps, Capsize: capsize, + CaptureSlotInUse: captureSlotsInUse(w.emitted, capsize), FcPrefix: fcPrefix, BmPrefix: bmPrefix, Anchors: getAnchors(tree), RightToLeft: rtl, FindOptimizations: tree.FindOptimizations, + LeftContextRunes: AnalyzeLeftContext(tree.Root), }, nil } +// AnalyzeLeftContext returns how many runes before a candidate start the matcher +// may inspect. Slicing the input down to that candidate is legal only when no +// opcode depends on the original search origin or unbounded left text. +// +// 0 never looks left of the start position +// 1 one previous rune is enough for \b, or to keep ^/\A from seeing +// the candidate as the original start (they test leftchars()==0) +// -1 do not slice: lookbehind, or \G (NtStart), which keys off textstart +func AnalyzeLeftContext(n *RegexNode) int { + if n == nil { + return 0 + } + + need := 0 + switch n.T { + case NtPosLook, NtNegLook: + if n.Options&RightToLeft != 0 { + return -1 + } + case NtStart: + return -1 + case NtBol, NtBeginning, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary: + need = 1 + } + + for _, child := range n.Children { + childNeed := AnalyzeLeftContext(child) + if childNeed < 0 { + return -1 + } + if childNeed > need { + need = childNeed + } + } + return need +} + // The main RegexCode generator. It does a depth-first walk // through the tree and calls EmitFragment to emits code before // and after each child of an interior node, and at each leaf. @@ -171,13 +275,37 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int) case NtConcatenate | BeforeChild, NtConcatenate | AfterChild, NtEmpty: case NtAlternate | BeforeChild: - if curIndex < len(node.Children)-1 { + if sets, leaders, _, ok := w.getDispatchCandidates(node); ok { + if curIndex == 0 { + w.dispatchCodes[node] = w.dispatchCode(sets) + w.emit1(Dispatch|bits, w.dispatchCodes[node]) + } + w.preconsumed[leaders[curIndex]] = true + w.patchDispatchBranch(node, curIndex, w.curPos()) + } else if curIndex < len(node.Children)-1 { w.pushInt(w.curPos()) w.emit1(Lazybranch, 0) } case NtAlternate | AfterChild: - if curIndex < len(node.Children)-1 { + if _, _, complete, ok := w.getDispatchCandidates(node); ok { + if curIndex < len(node.Children)-1 && !complete[curIndex] { + gotoPos := w.curPos() + w.emit1(Goto, 0) + w.dispatchGotos[node] = append(w.dispatchGotos[node], gotoPos) + } + if curIndex == len(node.Children)-1 { + end := w.curPos() + for _, gotoPos := range w.dispatchGotos[node] { + w.patchJump(gotoPos, end) + } + for branch, isComplete := range complete { + if isComplete { + w.patchDispatchBranch(node, branch, end) + } + } + } + } else if curIndex < len(node.Children)-1 { lbPos := w.popInt() w.pushInt(w.curPos()) w.emit1(Goto, 0) @@ -281,10 +409,14 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int) case NtGroup | BeforeChild, NtGroup | AfterChild: case NtCapture | BeforeChild: - w.emit(Setmark) + if w.emitCapture(node) { + w.emit(Setmark) + } case NtCapture | AfterChild: - w.emit2(Capturemark, w.mapCapnum(node.M), w.mapCapnum(node.N)) + if w.emitCapture(node) { + w.emit2(Capturemark, w.mapCapnum(node.M), w.mapCapnum(node.N)) + } case NtPosLook | BeforeChild: // NOTE: the following line causes lookahead/lookbehind to be @@ -317,7 +449,9 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int) w.emit(Forejump) case NtOne, NtNotone: - w.emit1(InstOp(node.T|ntBits), int(node.Ch)) + if !w.preconsumed[node] { + w.emit1(InstOp(node.T|ntBits), int(node.Ch)) + } case NtNotoneloop, NtNotoneloopatomic, NtNotonelazy, NtOneloop, NtOneloopatomic, NtOnelazy: if node.M > 0 { @@ -348,10 +482,22 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int) } case NtMulti: - w.emit1(InstOp(node.T|ntBits), w.stringCode(node.Str)) + str := node.Str + if w.preconsumed[node] { + if bits&Rtl != 0 { + str = str[:len(str)-1] + } else { + str = str[1:] + } + } + if len(str) > 0 { + w.emit1(InstOp(node.T|ntBits), w.stringCode(str)) + } case NtSet: - w.emit1(InstOp(node.T|ntBits), w.setCode(node.Set)) + if !w.preconsumed[node] { + w.emit1(InstOp(node.T|ntBits), w.setCode(node.Set)) + } case NtRef: w.emit1(InstOp(node.T|ntBits), w.mapCapnum(node.M)) @@ -359,6 +505,9 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int) case NtNothing, NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary, NtBeginning, NtStart, NtEndZ, NtEnd, NtUpdateBumpalong: w.emit(InstOp(node.T)) + case NtGrapheme: + w.emit(InstOp(node.T | ntBits)) + default: return fmt.Errorf("unexpected opcode in regular expression generation: %v", nodetype) } @@ -366,6 +515,17 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int) return nil } +func (w *writer) emitCapture(node *RegexNode) bool { + if w.quickCaptureSlots == nil { + return true + } + capnum, uncapnum := w.mapCapnum(node.M), w.mapCapnum(node.N) + if uncapnum != -1 { + return true + } + return capnum >= 0 && (capnum >= len(w.quickCaptureSlots) || w.quickCaptureSlots[capnum]) +} + // To avoid recursion, we use a simple integer stack. // This is the push. func (w *writer) pushInt(i int) { @@ -398,6 +558,53 @@ func (w *writer) patchJump(offset, jumpDest int) { w.emitted[offset+1] = jumpDest } +func (w *writer) getDispatchCandidates(node *RegexNode) ([]*CharSet, []*RegexNode, []bool, bool) { + if info, found := w.dispatchInfo[node]; found { + return info.sets, info.leaders, info.complete, info.ok + } + sets, leaders, complete, ok := node.dispatchCandidates() + w.dispatchInfo[node] = dispatchInfo{sets: sets, leaders: leaders, complete: complete, ok: ok} + return sets, leaders, complete, ok +} + +func (w *writer) dispatchCode(sets []*CharSet) int { + if w.counting { + return 0 + } + table := DispatchTable{ + Sets: make([]int, len(sets)), + Branches: make([]int, len(sets)), + } + if len(sets) >= 4 { + table.ASCII = &[128]uint16{} + } else { + table.ASCIIMasks = make([]uint64, len(sets)*2) + } + for i, set := range sets { + table.Sets[i] = w.setCode(set) + for ch := rune(0); ch < 128; ch++ { + if set.CharIn(ch) { + if table.ASCII != nil { + table.ASCII[ch] = uint16(i + 1) + } else { + table.ASCIIMasks[i*2+int(ch>>6)] |= uint64(1) << (ch & 63) + } + } + } + } + index := len(w.dispatchtable) + w.dispatchtable = append(w.dispatchtable, table) + return index +} + +func (w *writer) patchDispatchBranch(node *RegexNode, branch, target int) { + if w.counting { + return + } + table := w.dispatchCodes[node] + w.dispatchtable[table].Branches[branch] = target +} + // Returns an index in the set table for a charset // uses a map to eliminate duplicates. func (w *writer) setCode(set *CharSet) int { diff --git a/backend/vendor/github.com/go-pkgz/expirable-cache/v3/cache.go b/backend/vendor/github.com/go-pkgz/expirable-cache/v3/cache.go index 4b6d9b4e..a8f50b3b 100644 --- a/backend/vendor/github.com/go-pkgz/expirable-cache/v3/cache.go +++ b/backend/vendor/github.com/go-pkgz/expirable-cache/v3/cache.go @@ -8,6 +8,9 @@ // In case of default TTL (10 years) and default MaxSize (0, unlimited) the cache will be truly unlimited // and will never delete entries from itself automatically. // +// Get, Peek, Contains and Values treat an expired entry as missing, while Len and Keys report +// everything the cache still holds, expired entries included. +// // Important: only reliable way of not having expired entries stuck in a cache is to // run cache.DeleteExpired periodically using time.Ticker, advisable period is 1/2 of TTL. package cache @@ -78,9 +81,10 @@ func NewCache[K comparable, V any]() Cache[K, V] { } } -// Add adds a value to the cache. Returns true if an eviction occurred. +// Add adds a value to the cache. Returns true if an eviction occurred, either +// because the size was exceeded or because the oldest entry expired. // Returns false if there was no eviction: the item was already in the cache, -// or the size was not exceeded. +// or nothing had to be removed. func (c *cacheImpl[K, V]) Add(key K, value V) (evicted bool) { return c.addWithTTL(key, value, c.ttl) } @@ -90,9 +94,10 @@ func (c *cacheImpl[K, V]) Set(key K, value V, ttl time.Duration) { c.addWithTTL(key, value, ttl) } -// Returns true if an eviction occurred. +// Returns true if an eviction occurred, either because the size was exceeded +// or because the oldest entry expired. // Returns false if there was no eviction: the item was already in the cache, -// or the size was not exceeded. +// or nothing had to be removed. func (c *cacheImpl[K, V]) addWithTTL(key K, value V, ttl time.Duration) (evicted bool) { if ttl == 0 { ttl = c.ttl @@ -119,15 +124,16 @@ func (c *cacheImpl[K, V]) addWithTTL(key K, value V, ttl time.Duration) (evicted ent := c.evictList.Back() if ent != nil && now.After(ent.Value.(*cacheItem[K, V]).expiresAt) { c.removeElement(ent) + evicted = true } } - evict := c.maxKeys > 0 && len(c.items) > c.maxKeys // Verify size not exceeded - if evict { + if c.maxKeys > 0 && len(c.items) > c.maxKeys { c.removeOldest() + evicted = true } - return evict + return evicted } // Get returns the key value if it's not expired @@ -152,12 +158,15 @@ func (c *cacheImpl[K, V]) Get(key K) (V, bool) { } // Contains checks if a key is in the cache, without updating the recent-ness -// or deleting it for being stale. +// or deleting it for being stale. Expired entries are reported as missing. func (c *cacheImpl[K, V]) Contains(key K) (ok bool) { c.Lock() defer c.Unlock() - _, ok = c.items[key] - return ok + ent, ok := c.items[key] + if !ok { + return false + } + return !time.Now().After(ent.Value.(*cacheItem[K, V]).expiresAt) } // Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key. @@ -199,10 +208,10 @@ func (c *cacheImpl[K, V]) Keys() []K { // Values returns a slice of the values in the cache, from oldest to newest. // Expired entries are filtered out. func (c *cacheImpl[K, V]) Values() []V { - values := make([]V, 0, len(c.items)) now := time.Now() c.Lock() defer c.Unlock() + values := make([]V, 0, len(c.items)) for ent := c.evictList.Back(); ent != nil; ent = ent.Prev() { if !now.After(ent.Value.(*cacheItem[K, V]).expiresAt) { values = append(values, ent.Value.(*cacheItem[K, V]).value) diff --git a/backend/vendor/github.com/go-pkgz/jrpc/.gitignore b/backend/vendor/github.com/go-pkgz/jrpc/.gitignore index f1c181ec..da1046f2 100644 --- a/backend/vendor/github.com/go-pkgz/jrpc/.gitignore +++ b/backend/vendor/github.com/go-pkgz/jrpc/.gitignore @@ -10,3 +10,7 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out + +# Example binaries, produced by `go build` in the example modules +/_example/application/application +/_example/plugin/plugin diff --git a/backend/vendor/github.com/go-pkgz/jrpc/README.md b/backend/vendor/github.com/go-pkgz/jrpc/README.md index 5a5dec3e..32a0b460 100644 --- a/backend/vendor/github.com/go-pkgz/jrpc/README.md +++ b/backend/vendor/github.com/go-pkgz/jrpc/README.md @@ -15,39 +15,55 @@ type Plugin struct { } // create plugin (jrpc server) with NewServer where required param is a base url for rpc calls -plugin := NewServer("/command") +plugin := jrpc.NewServer("/command") -// then add you function to map -plugin.Add("mycommand", func(id uint64, params json.RawMessage) Response { +// then add your function to map +plugin.Add("mycommand", func(id uint64, params json.RawMessage) jrpc.Response { return jrpc.EncodeResponse(id, "hello, it works", nil) }) // and run server with port number value -plugin.Run(9090) +plugin.Run(8080) ``` -The constructor `NewServer` accept two parameters: +The constructor `NewServer` accepts two parameters: * `API` - a base url for rpc calls -* `Options` - optional parameters such is timeouts, logger, limits, middlewares and etc. - * `Auth` - set credentials basic auth to server, accepts `username` and `password` - * `WithTimeout` - sets global timeouts for server requests, such as read, write and idle. Call accept `Timeouts` struct. - * `WithLimits` - define limit for server call, accepts limit value in `float64` type - * `WithThrottler` - sets throttler middleware with specify limit value - * `WithtSignature` - sets server signature, accept appName, author and version. Disable by default. - * `WithLogger` - define custom logger (e.g. [lgr](https://github.com/go-pkgz/lgr)) - * `WithMiddlewares` - sets custom middlewares list to server, accepts list of handler with idiomatic type `func(http.Handler) http.Handler` +* `Options` - optional parameters such as timeouts, logger, limits, middlewares and so on. + * `Auth` - sets basic auth credentials, accepts `username` and `password`. Auth is enforced only if both of them + set to non-empty values; setting just one leaves the server serving every request unauthenticated + * `WithTimeouts` - sets server timeouts, accepts a `Timeouts` struct with `ReadHeaderTimeout`, `WriteTimeout`, + `IdleTimeout` and `CallTimeout`. `CallTimeout` limits the time allowed for a single call and responds with `503` if + exceeded, and has to be set below `WriteTimeout`, otherwise the write deadline kills the connection before the + `503` can be sent + * `WithLimits` - defines a limit of calls/sec per client, accepts limit value in `float64` type + * `WithThrottler` - sets throttler middleware limiting the number of parallel calls to the server + * `WithSignature` - sets server signature, accepts appName, author and version. Disabled by default + * `WithLogger` - defines custom logger (e.g. [lgr](https://github.com/go-pkgz/lgr)) + * `WithMiddlewares` - sets custom middlewares list to server, accepts list of handlers with idiomatic type `func(http.Handler) http.Handler` Example with options: ```go -plugin := NewServer("/command", - Auth("user", "password"), - WithTimeout(Timeouts{ReadHeaderTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second}), - WithThrottler(120), - WithLimits(100), - WithtSignature("the best plugin ever", "author", "1.0.0"), - WithMiddlewares(middleware.Heartbeat('/ping'), middleware.Profiler, middleware.StripSlashes), +import ( + "time" + + "github.com/go-pkgz/jrpc" + "github.com/go-pkgz/rest" ) -``` + +plugin := jrpc.NewServer("/command", + jrpc.Auth("user", "password"), + jrpc.WithTimeouts(jrpc.Timeouts{ + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 10 * time.Second, + CallTimeout: 25 * time.Second, + }), + jrpc.WithThrottler(120), + jrpc.WithLimits(100), + jrpc.WithSignature("the best plugin ever", "author", "1.0.0"), + jrpc.WithMiddlewares(rest.Trace), +) +``` ### Application (client) @@ -67,8 +83,42 @@ if err = json.Unmarshal(*resp.Result, &message); err != nil { } ``` -*for functional examples for both plugin and application see [_example](https://github.com/go-pkgz/jrpc/tree/master/_example)* - +### Running the example + +[_example](https://github.com/go-pkgz/jrpc/tree/master/_example) has a working pair of a plugin and an application. +Both are separate go modules pointing to the local jrpc with a `replace` directive, so no extra setup is needed +beyond go 1.24 or later and a free local port 8080. Start the plugin first, in one terminal: + +```sh +cd _example/plugin +go run . +``` + +It registers two handlers and listens on port 8080: + +``` +[INFO] add handler for store.save +[INFO] add handler for store.load +[INFO] listen on [::]:8080 +``` + +Then run the application in another terminal: + +```sh +cd _example/application +go run . +``` + +It calls the plugin three times and prints the results: + +``` +stored {TS:2025-01-12 12:00:00 +0000 UTC Value:12345} with id=54118548792 +loaded {TS:2025-01-12 12:00:00 +0000 UTC Value:12345} from id=54118548792 +can't load for id=something, not found +``` + +The application exits on its own, the plugin keeps listening until stopped with Ctrl-C. + ## Technical details * `jrpc.Server` runs on user-defined port as a regular http server @@ -95,7 +145,9 @@ if err = json.Unmarshal(*resp.Result, &message); err != nil { * Params can be a struct, primitive type or slice of values, even with different types. * Server defines `ServerFn` handler function to react on a POST request. The handler provided by the user. -* Communication between the server and the caller can be protected with basic auth. +* Communication between the server and the caller can be protected with basic auth. The protection is on only if + both user and password set with the `Auth` option; with either of them empty the server responds to every request + without asking for credentials. * [Client](https://github.com/go-pkgz/jrpc/blob/master/client.go) provides a single method `Call` and return `Response`
response details: diff --git a/backend/vendor/github.com/go-pkgz/jrpc/client.go b/backend/vendor/github.com/go-pkgz/jrpc/client.go index 0a516306..8ac4aa13 100644 --- a/backend/vendor/github.com/go-pkgz/jrpc/client.go +++ b/backend/vendor/github.com/go-pkgz/jrpc/client.go @@ -23,7 +23,7 @@ type Client struct { // Empty args will be ignored, single arg will be marshaled as-us and multiple args marshaled as []interface{}. // Returns Response and error. Note: Response has it's own Error field, but that onw controlled by server. // Returned error represent client-level errors, like failed http call, failed marshaling and so on. -func (r *Client) Call(method string, args ...interface{}) (*Response, error) { +func (r *Client) Call(method string, args ...any) (*Response, error) { var b []byte var err error diff --git a/backend/vendor/github.com/go-pkgz/jrpc/jrpc.go b/backend/vendor/github.com/go-pkgz/jrpc/jrpc.go index 86063690..91c29a5c 100644 --- a/backend/vendor/github.com/go-pkgz/jrpc/jrpc.go +++ b/backend/vendor/github.com/go-pkgz/jrpc/jrpc.go @@ -10,9 +10,9 @@ import ( // Request encloses method name and all params type Request struct { - Method string `json:"method"` // method (function) name - Params interface{} `json:"params,omitempty"` // function arguments - ID uint64 `json:"id"` // unique call id + Method string `json:"method"` // method (function) name + Params any `json:"params,omitempty"` // function arguments + ID uint64 `json:"id"` // unique call id } // Response encloses result and error received from remote server @@ -23,7 +23,7 @@ type Response struct { } // EncodeResponse convert anything (type interface{}) and incoming error (if any) to Response -func EncodeResponse(id uint64, resp interface{}, e error) Response { +func EncodeResponse(id uint64, resp any, e error) Response { v, err := json.Marshal(&resp) if err != nil { return Response{Error: err.Error()} diff --git a/backend/vendor/github.com/go-pkgz/jrpc/options.go b/backend/vendor/github.com/go-pkgz/jrpc/options.go index e2c4e28d..528e4363 100644 --- a/backend/vendor/github.com/go-pkgz/jrpc/options.go +++ b/backend/vendor/github.com/go-pkgz/jrpc/options.go @@ -7,7 +7,9 @@ import ( // Option func type type Option func(s *Server) -// Auth sets basic auth credentials, required +// Auth sets basic auth credentials, optional. +// Auth enforced only if both user and password set to non-empty values, otherwise the server +// keeps serving every request unauthenticated. Setting just one of them doesn't enable auth. func Auth(user, password string) Option { return func(s *Server) { s.authUser = user diff --git a/backend/vendor/github.com/go-pkgz/jrpc/server.go b/backend/vendor/github.com/go-pkgz/jrpc/server.go index e32a73e3..225e1853 100644 --- a/backend/vendor/github.com/go-pkgz/jrpc/server.go +++ b/backend/vendor/github.com/go-pkgz/jrpc/server.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net" "net/http" "sync" "time" @@ -13,12 +14,13 @@ import ( "github.com/go-pkgz/routegroup" ) -// Server is json-rpc server with an optional basic auth +// Server is json-rpc server with an optional basic auth. +// Auth enforced only if both authUser and authPasswd set, see Auth option. type Server struct { api string // url path, i.e. "/command" or "/rpc" etc., required - authUser string // basic auth user name, should match Client.AuthUser, optional - authPasswd string // basic auth password, should match Client.AuthPasswd, optional + authUser string // basic auth user name, should match Client.AuthUser, optional, no auth if empty + authPasswd string // basic auth password, should match Client.AuthPasswd, optional, no auth if empty customMiddlewares middlewares // list of custom middlewares, should match array of http.Handler func, optional signature signaturePayload // add server signature to server response headers appName, author, version), disable by default @@ -81,15 +83,29 @@ func NewServer(api string, options ...Option) *Server { return srv } -// Run http server on given port +// Run http server on given port, blocks until Shutdown called or the server failed func (s *Server) Run(port int) error { - if s.authUser == "" || s.authPasswd == "" { - s.logger.Logf("[WARN] extension server runs without auth") + if len(s.funcs.m) == 0 { + return fmt.Errorf("nothing mapped for dispatch, Add has to be called prior to Run") } - if s.funcs.m == nil && len(s.funcs.m) == 0 { - return fmt.Errorf("nothing mapped for dispatch, Add has to be called prior to Run") + s.activate() + + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return fmt.Errorf("can't listen on port %d: %w", port, err) + } + + return s.serve(ln) +} + +// activate makes http server with all the middlewares and the dispatch handler. +// after this call Add won't accept new methods. +func (s *Server) activate() { + + if s.authUser == "" || s.authPasswd == "" { + s.logger.Logf("[WARN] extension server runs without auth, both user and password have to be set to enable it") } router := routegroup.New(http.NewServeMux()) @@ -124,16 +140,26 @@ func (s *Server) Run(port int) error { s.httpServer.Lock() s.httpServer.Server = &http.Server{ - Addr: fmt.Sprintf(":%d", port), Handler: router, ReadHeaderTimeout: s.timeouts.ReadHeaderTimeout, WriteTimeout: s.timeouts.WriteTimeout, IdleTimeout: s.timeouts.IdleTimeout, } s.httpServer.Unlock() +} - s.logger.Logf("[INFO] listen on %d", port) - return s.httpServer.ListenAndServe() +// serve runs activated http server on the provided listener +func (s *Server) serve(l net.Listener) error { + s.httpServer.Lock() + srv := s.httpServer.Server + s.httpServer.Unlock() + + if srv == nil { + return fmt.Errorf("server is not activated") + } + + s.logger.Logf("[INFO] listen on %s", l.Addr()) + return srv.Serve(l) } // Shutdown http server @@ -202,7 +228,8 @@ func (s *Server) handler(w http.ResponseWriter, r *http.Request) { rest.RenderJSON(w, fn(req.ID, params)) } -// basicAuth middleware. enabled only if both AuthUser and AuthPasswd defined. +// basicAuth middleware, enabled only if both authUser and authPasswd set to non-empty values. +// with either of them empty every request passes through unauthenticated. func (s *Server) basicAuth(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -229,30 +256,27 @@ func getDefaultTimeouts() Timeouts { } } -// timeout middleware cancels context after given duration +// timeout middleware limits the time allowed for the call, responds with 503 and drops +// the late handler writes if the deadline reached func timeout(dt time.Duration) func(http.Handler) http.Handler { return func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), dt) - defer cancel() - h.ServeHTTP(w, r.WithContext(ctx)) - }) + return http.TimeoutHandler(h, dt, `{"error":"call timeout"}`) } } // L defined logger interface used for an optional rest logging type L interface { - Logf(format string, args ...interface{}) + Logf(format string, args ...any) } // LoggerFunc type is an adapter to allow the use of ordinary functions as Logger. -type LoggerFunc func(format string, args ...interface{}) +type LoggerFunc func(format string, args ...any) // Logf calls f(id) -func (f LoggerFunc) Logf(format string, args ...interface{}) { f(format, args...) } +func (f LoggerFunc) Logf(format string, args ...any) { f(format, args...) } // NoOpLogger logger does nothing -var NoOpLogger = LoggerFunc(func(format string, args ...interface{}) {}) //nolint +var NoOpLogger = LoggerFunc(func(format string, args ...any) {}) //nolint // rateLimitByIP returns middleware that limits requests per second for each client IP. // Uses X-Real-IP header (set by rest.RealIP middleware) for client identification. diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/cache.go b/backend/vendor/github.com/go-pkgz/lcw/v2/cache.go index 58e259bb..818b457a 100644 --- a/backend/vendor/github.com/go-pkgz/lcw/v2/cache.go +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/cache.go @@ -11,11 +11,26 @@ import ( ) // Sizer allows to perform size-based restrictions, optional. -// If not defined both maxValueSize and maxCacheSize checks will be ignored +// Values implementing it define their own size, []byte and string are sized by length. +// For any other type both maxValueSize and maxCacheSize checks are ignored. type Sizer interface { Size() int } +// sizeOf returns the size of the value for size-based restrictions and reports +// whether the value can be sized at all. Sizer takes priority over the built-in types. +func sizeOf(value any) (int, bool) { + switch v := value.(type) { + case Sizer: + return v.Size(), true + case []byte: + return len(v), true + case string: + return len(v), true + } + return 0, false +} + // LoadingCache defines guava-like cache with Get method returning cached value ao retrieving it if not in cache type LoadingCache[V any] interface { Get(key string, fn func() (V, error)) (val V, err error) // load or get from cache diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/eventbus/redis.go b/backend/vendor/github.com/go-pkgz/lcw/v2/eventbus/redis.go index b97870c8..0b077bc7 100644 --- a/backend/vendor/github.com/go-pkgz/lcw/v2/eventbus/redis.go +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/eventbus/redis.go @@ -2,11 +2,11 @@ package eventbus import ( "context" + "errors" "fmt" "strings" "time" - "github.com/hashicorp/go-multierror" "github.com/redis/go-redis/v9" ) @@ -67,12 +67,12 @@ func (m *RedisPubSub) Publish(fromID, key string) error { func (m *RedisPubSub) Close() error { close(m.done) - errs := new(multierror.Error) + var errs []error if err := m.pubSub.Close(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("problem closing pubSub client: %w", err)) + errs = append(errs, fmt.Errorf("problem closing pubSub client: %w", err)) } if err := m.client.Close(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("problem closing redis client: %w", err)) + errs = append(errs, fmt.Errorf("problem closing redis client: %w", err)) } - return errs.ErrorOrNil() + return errors.Join(errs...) } diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/expirable_cache.go b/backend/vendor/github.com/go-pkgz/lcw/v2/expirable_cache.go index 6d64f4a9..9000300c 100644 --- a/backend/vendor/github.com/go-pkgz/lcw/v2/expirable_cache.go +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/expirable_cache.go @@ -18,9 +18,14 @@ type ExpirableCache[V any] struct { currentSize int64 id string backend *expirable.LRU[string, V] + loads loadGroup[V] } -// NewExpirableCache makes expirable LoadingCache implementation, 1000 max keys by default and 5m TTL +// NewExpirableCache makes expirable LoadingCache implementation, 1000 max keys by default and 5m TTL. +// Note that the underlying hashicorp/golang-lru expirable backend calls eviction callbacks while +// holding its own lock, so an OnEvicted handler must not call back into the same cache, it would deadlock. +// The same handler is safe to use with LruCache and with the v1 ExpirableCache. +// Reported upstream as https://github.com/hashicorp/golang-lru/issues/230 func NewExpirableCache[V any](opts ...Option[V]) (*ExpirableCache[V], error) { res := ExpirableCache[V]{ Workers: Workers[V]{ @@ -46,8 +51,7 @@ func NewExpirableCache[V any](opts ...Option[V]) (*ExpirableCache[V], error) { if res.onEvicted != nil { res.onEvicted(key, value) } - if s, ok := any(value).(Sizer); ok { - size := s.Size() + if size, ok := sizeOf(value); ok { atomic.AddInt64(&res.currentSize, -1*int64(size)) } // ignore the error on Publish as we don't have log inside the module and @@ -66,26 +70,36 @@ func (c *ExpirableCache[V]) Get(key string, fn func() (V, error)) (data V, err e return v, nil } - if data, err = fn(); err != nil { - atomic.AddInt64(&c.Errors, 1) - return data, err - } - atomic.AddInt64(&c.Misses, 1) + // concurrent callers for the same key wait for the first load instead of loading on their own, + // otherwise each of them would add the value and count its size again + return c.loads.do(key, func() (V, error) { + if v, ok := c.backend.Get(key); ok { // filled by the load we were waiting for + atomic.AddInt64(&c.Hits, 1) + return v, nil + } - if !c.allowed(key, data) { - return data, nil - } + data, err := fn() + if err != nil { + atomic.AddInt64(&c.Errors, 1) + return data, err + } + atomic.AddInt64(&c.Misses, 1) - if s, ok := any(data).(Sizer); ok { - if c.maxCacheSize > 0 && atomic.LoadInt64(&c.currentSize)+int64(s.Size()) >= c.maxCacheSize { + if !c.allowed(key, data) { return data, nil } - atomic.AddInt64(&c.currentSize, int64(s.Size())) - } - c.backend.Add(key, data) + if size, ok := sizeOf(data); ok { + if c.maxCacheSize > 0 && atomic.LoadInt64(&c.currentSize)+int64(size) >= c.maxCacheSize { + return data, nil + } + atomic.AddInt64(&c.currentSize, int64(size)) + } - return data, nil + c.backend.Add(key, data) + + return data, nil + }) } // Invalidate removes keys with passed predicate fn, i.e. fn(key) should be true to get evicted @@ -154,14 +168,14 @@ func (c *ExpirableCache[V]) keys() int { } func (c *ExpirableCache[V]) allowed(key string, data V) bool { - if c.backend.Len() >= c.maxKeys { + if c.maxKeys > 0 && c.backend.Len() >= c.maxKeys { return false } if c.maxKeySize > 0 && len(key) > c.maxKeySize { return false } - if s, ok := any(data).(Sizer); ok { - if c.maxValueSize > 0 && s.Size() >= c.maxValueSize { + if size, ok := sizeOf(data); ok { + if c.maxValueSize > 0 && size >= c.maxValueSize { return false } } diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/loadgroup.go b/backend/vendor/github.com/go-pkgz/lcw/v2/loadgroup.go new file mode 100644 index 00000000..a5883cb5 --- /dev/null +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/loadgroup.go @@ -0,0 +1,69 @@ +package lcw + +import ( + "errors" + "sync" +) + +// ErrLoaderPanic is what callers waiting for a load get when the loader panicked. +// The panic value itself keeps propagating in the goroutine that ran the loader. +var ErrLoaderPanic = errors.New("cache loader panic") + +// loadGroup makes sure only one load function per key runs at a time. +// concurrent calls for the same key wait for the in-flight one and share its result, +// so a cold key is loaded once instead of once per caller. +// the loader must not call Get for the same key on the same cache, it would wait for itself. +type loadGroup[V any] struct { + mu sync.Mutex + calls map[string]*loadCall[V] +} + +type loadCall[V any] struct { + wg sync.WaitGroup + val V + err error +} + +// do calls fn for the given key unless the same key is already being loaded, +// in that case it waits for the in-flight load and returns its result. +func (g *loadGroup[V]) do(key string, fn func() (V, error)) (V, error) { + g.mu.Lock() + if g.calls == nil { + g.calls = make(map[string]*loadCall[V]) + } + if call, ok := g.calls[key]; ok { + g.mu.Unlock() + call.wg.Wait() + return call.val, call.err + } + + call := &loadCall[V]{} + call.wg.Add(1) + g.calls[key] = call + g.mu.Unlock() + + // waiters are released even if fn panics, and get an error instead of a zero value + // with no error at all. The panic keeps propagating in the calling goroutine, as it + // would without the load coordination. + defer func() { + if p := recover(); p != nil { + // a fixed error, formatting the panic value here would run arbitrary + // code of its Error or String method before the waiters are released + call.err = ErrLoaderPanic + g.done(key, call) + panic(p) + } + g.done(key, call) + }() + + call.val, call.err = fn() + return call.val, call.err +} + +// done drops the in-flight call and releases everybody waiting for it +func (g *loadGroup[V]) done(key string, call *loadCall[V]) { + g.mu.Lock() + delete(g.calls, key) + g.mu.Unlock() + call.wg.Done() +} diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/lru_cache.go b/backend/vendor/github.com/go-pkgz/lcw/v2/lru_cache.go index 9c63490b..a25d0e90 100644 --- a/backend/vendor/github.com/go-pkgz/lcw/v2/lru_cache.go +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/lru_cache.go @@ -2,6 +2,7 @@ package lcw import ( "fmt" + "math" "sync/atomic" "github.com/google/uuid" @@ -17,6 +18,7 @@ type LruCache[V any] struct { backend *lru.Cache[string, V] currentSize int64 id string // uuid identifying cache instance + loads loadGroup[V] } // NewLruCache makes LRU LoadingCache implementation, 1000 max keys by default @@ -48,8 +50,7 @@ func (c *LruCache[V]) init() error { if c.onEvicted != nil { c.onEvicted(key, value) } - if s, ok := any(value).(Sizer); ok { - size := s.Size() + if size, ok := sizeOf(value); ok { atomic.AddInt64(&c.currentSize, -1*int64(size)) } _ = c.eventBus.Publish(c.id, key) // signal invalidation to other nodes @@ -57,7 +58,11 @@ func (c *LruCache[V]) init() error { var err error // OnEvicted called automatically for expired and manually deleted - if c.backend, err = lru.NewWithEvict[string, V](c.maxKeys, onEvicted); err != nil { + maxKeys := c.maxKeys + if maxKeys <= 0 { // 0 means unlimited, lru backend requires a positive size + maxKeys = math.MaxInt + } + if c.backend, err = lru.NewWithEvict[string, V](maxKeys, onEvicted); err != nil { return fmt.Errorf("failed to make lru cache backend: %w", err) } @@ -71,29 +76,39 @@ func (c *LruCache[V]) Get(key string, fn func() (V, error)) (data V, err error) return v, nil } - if data, err = fn(); err != nil { - atomic.AddInt64(&c.Errors, 1) - return data, err - } + // concurrent callers for the same key wait for the first load instead of loading on their own, + // otherwise each of them would add the value and count its size again + return c.loads.do(key, func() (V, error) { + if v, ok := c.backend.Get(key); ok { // filled by the load we were waiting for + atomic.AddInt64(&c.Hits, 1) + return v, nil + } - atomic.AddInt64(&c.Misses, 1) + data, err := fn() + if err != nil { + atomic.AddInt64(&c.Errors, 1) + return data, err + } - if !c.allowed(key, data) { - return data, nil - } + atomic.AddInt64(&c.Misses, 1) - c.backend.Add(key, data) + if !c.allowed(key, data) { + return data, nil + } - if s, ok := any(data).(Sizer); ok { - atomic.AddInt64(&c.currentSize, int64(s.Size())) - if c.maxCacheSize > 0 && atomic.LoadInt64(&c.currentSize) > c.maxCacheSize { - for atomic.LoadInt64(&c.currentSize) > c.maxCacheSize { - c.backend.RemoveOldest() + c.backend.Add(key, data) + + if size, ok := sizeOf(data); ok { + atomic.AddInt64(&c.currentSize, int64(size)) + for c.maxCacheSize > 0 && atomic.LoadInt64(&c.currentSize) > c.maxCacheSize { + if _, _, ok := c.backend.RemoveOldest(); !ok { // nothing left to evict + break + } } } - } - return data, nil + return data, nil + }) } // Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key. @@ -161,8 +176,8 @@ func (c *LruCache[V]) allowed(key string, data V) bool { if c.maxKeySize > 0 && len(key) > c.maxKeySize { return false } - if s, ok := any(data).(Sizer); ok { - if c.maxValueSize > 0 && s.Size() >= c.maxValueSize { + if size, ok := sizeOf(data); ok { + if c.maxValueSize > 0 && size >= c.maxValueSize { return false } } diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/options.go b/backend/vendor/github.com/go-pkgz/lcw/v2/options.go index c960d13e..dad576b4 100644 --- a/backend/vendor/github.com/go-pkgz/lcw/v2/options.go +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/options.go @@ -7,6 +7,7 @@ import ( "github.com/go-pkgz/lcw/v2/eventbus" ) +// Workers holds cache configuration options type Workers[V any] struct { maxKeys int maxValueSize int @@ -16,6 +17,8 @@ type Workers[V any] struct { onEvicted func(key string, value V) eventBus eventbus.PubSub strToV func(string) V + + redisKeyPrefix string } // Option func type @@ -29,50 +32,53 @@ func NewOpts[T any]() *WorkerOptions[T] { return &WorkerOptions[T]{} } -// MaxValSize functional option defines the largest value's size allowed to be cached +// MaxValSize functional option defines the largest value's size allowed to be cached. +// Applies to values implementing Sizer as well as []byte and string, other types are not limited. // By default it is 0, which means unlimited. -func (o *WorkerOptions[V]) MaxValSize(max int) Option[V] { +func (o *WorkerOptions[V]) MaxValSize(maximum int) Option[V] { return func(o *Workers[V]) error { - if max < 0 { + if maximum < 0 { return fmt.Errorf("negative max value size") } - o.maxValueSize = max + o.maxValueSize = maximum return nil } } // MaxKeySize functional option defines the largest key's size allowed to be used in cache // By default it is 0, which means unlimited. -func (o *WorkerOptions[V]) MaxKeySize(max int) Option[V] { +func (o *WorkerOptions[V]) MaxKeySize(maximum int) Option[V] { return func(o *Workers[V]) error { - if max < 0 { + if maximum < 0 { return fmt.Errorf("negative max key size") } - o.maxKeySize = max + o.maxKeySize = maximum return nil } } // MaxKeys functional option defines how many keys to keep. // By default, it is 0, which means unlimited. -func (o *WorkerOptions[V]) MaxKeys(max int) Option[V] { +func (o *WorkerOptions[V]) MaxKeys(maximum int) Option[V] { return func(o *Workers[V]) error { - if max < 0 { + if maximum < 0 { return fmt.Errorf("negative max keys") } - o.maxKeys = max + o.maxKeys = maximum return nil } } // MaxCacheSize functional option defines the total size of cached data. +// Applies to values implementing Sizer as well as []byte and string, other types are not counted. +// Not supported by RedisCache, which accepts the option but ignores it. // By default, it is 0, which means unlimited. -func (o *WorkerOptions[V]) MaxCacheSize(max int64) Option[V] { +func (o *WorkerOptions[V]) MaxCacheSize(maximum int64) Option[V] { return func(o *Workers[V]) error { - if max < 0 { + if maximum < 0 { return fmt.Errorf("negative max cache size") } - o.maxCacheSize = max + o.maxCacheSize = maximum return nil } } @@ -89,7 +95,9 @@ func (o *WorkerOptions[V]) TTL(ttl time.Duration) Option[V] { } } -// OnEvicted sets callback on invalidation event +// OnEvicted sets callback on invalidation event. +// The callback runs outside the cache lock for LruCache, but ExpirableCache calls it while its +// backend lock is held, so with ExpirableCache the handler must not call back into the same cache. func (o *WorkerOptions[V]) OnEvicted(fn func(key string, value V)) Option[V] { return func(o *Workers[V]) error { o.onEvicted = fn @@ -105,6 +113,18 @@ func (o *WorkerOptions[V]) EventBus(pubSub eventbus.PubSub) Option[V] { } } +// RedisKeyPrefix sets the prefix for all keys stored in redis, making the cache +// use only its own namespace instead of the whole redis database. +// Applies to RedisCache only, ignored by other backends. Empty by default, which means +// the cache assumes exclusive ownership of the selected redis database. +// Note that with a prefix set and MaxKeys defined, key counting scans the keyspace on every miss. +func (o *WorkerOptions[V]) RedisKeyPrefix(prefix string) Option[V] { + return func(o *Workers[V]) error { + o.redisKeyPrefix = prefix + return nil + } +} + // StrToV sets strToV function for RedisCache func (o *WorkerOptions[V]) StrToV(fn func(string) V) Option[V] { return func(o *Workers[V]) error { diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/redis_cache.go b/backend/vendor/github.com/go-pkgz/lcw/v2/redis_cache.go index 732ddf31..056d6cee 100644 --- a/backend/vendor/github.com/go-pkgz/lcw/v2/redis_cache.go +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/redis_cache.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "strings" "sync/atomic" "time" @@ -18,12 +19,16 @@ const RedisValueSizeLimit = 512 * 1024 * 1024 type RedisCache[V any] struct { Workers[V] CacheStat - backend *redis.Client + backend redis.UniversalClient + loads loadGroup[V] } // NewRedisCache makes Redis LoadingCache implementation. // Supports only string and string-based types and will return error otherwise. -func NewRedisCache[V any](backend *redis.Client, opts ...Option[V]) (*RedisCache[V], error) { +// Without RedisKeyPrefix the cache assumes exclusive ownership of the selected Redis database, +// i.e. Purge flushes it entirely and Keys, Stat and MaxKeys count every key in it. +// MaxCacheSize is not supported by this backend, it is accepted but ignored, and Stat reports size 0. +func NewRedisCache[V any](backend redis.UniversalClient, opts ...Option[V]) (*RedisCache[V], error) { // check if V is string, not underlying type but directly, and otherwise return error if strToV is nil as it should be defined res := RedisCache[V]{ @@ -60,54 +65,78 @@ func NewRedisCache[V any](backend *redis.Client, opts ...Option[V]) (*RedisCache return &res, nil } +// fullKey makes the physical redis key for the given logical key +func (c *RedisCache[V]) fullKey(key string) string { + return c.redisKeyPrefix + key +} + +// scanKeys returns all physical keys belonging to this cache +func (c *RedisCache[V]) scanKeys() []string { + return c.backend.Keys(context.Background(), escapeGlob(c.redisKeyPrefix)+"*").Val() +} + // Get gets value by key or load with fn if not found in cache func (c *RedisCache[V]) Get(key string, fn func() (V, error)) (data V, err error) { - v, getErr := c.backend.Get(context.Background(), key).Result() + v, getErr := c.backend.Get(context.Background(), c.fullKey(key)).Result() switch { // RedisClient returns nil when find a key in DB case getErr == nil: atomic.AddInt64(&c.Hits, 1) - switch any(data).(type) { - case string: - return any(v).(V), nil - default: - return c.strToV(v), nil - } - // RedisClient returns redis.Nil when doesn't find a key in DB + return c.toV(v), nil + // RedisClient returns redis.Nil when doesn't find a key in DB, load it below case errors.Is(getErr, redis.Nil): - if data, err = fn(); err != nil { - atomic.AddInt64(&c.Errors, 1) - return data, err - } // RedisClient returns !nil when something goes wrong while get data default: atomic.AddInt64(&c.Errors, 1) - switch any(data).(type) { - case string: - return any(v).(V), getErr - default: - return c.strToV(v), getErr + return c.toV(v), getErr + } + + // concurrent callers for the same key wait for the first load instead of loading on their own + return c.loads.do(key, func() (V, error) { + cached, cachedErr := c.backend.Get(context.Background(), c.fullKey(key)).Result() + switch { + case cachedErr == nil: + atomic.AddInt64(&c.Hits, 1) // filled by the load we were waiting for + return c.toV(cached), nil + case !errors.Is(cachedErr, redis.Nil): // a broken backend is not a miss, same as above + atomic.AddInt64(&c.Errors, 1) + var empty V // cached holds no value here, no point running StrToV over it + return empty, cachedErr + } + + data, err := fn() + if err != nil { + atomic.AddInt64(&c.Errors, 1) + return data, err + } + atomic.AddInt64(&c.Misses, 1) + + if !c.allowed(key, data) { + return data, nil + } + + if _, setErr := c.backend.Set(context.Background(), c.fullKey(key), data, c.ttl).Result(); setErr != nil { + atomic.AddInt64(&c.Errors, 1) + return data, setErr } - } - atomic.AddInt64(&c.Misses, 1) - if !c.allowed(key, data) { return data, nil - } + }) +} - _, setErr := c.backend.Set(context.Background(), key, data, c.ttl).Result() - if setErr != nil { - atomic.AddInt64(&c.Errors, 1) - return data, setErr +// toV converts the string stored in redis back to V, directly for string and via StrToV for string-based types +func (c *RedisCache[V]) toV(s string) V { + var v V + if _, ok := any(v).(string); ok { + return any(s).(V) } - - return data, nil + return c.strToV(s) } // Invalidate removes keys with passed predicate fn, i.e. fn(key) should be true to get evicted func (c *RedisCache[V]) Invalidate(fn func(key string) bool) { - for _, key := range c.backend.Keys(context.Background(), "*").Val() { // Keys() returns copy of cache's key, safe to remove directly - if fn(key) { + for _, key := range c.scanKeys() { // Keys() returns copy of cache's key, safe to remove directly + if fn(strings.TrimPrefix(key, c.redisKeyPrefix)) { c.backend.Del(context.Background(), key) } } @@ -115,33 +144,43 @@ func (c *RedisCache[V]) Invalidate(fn func(key string) bool) { // Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key. func (c *RedisCache[V]) Peek(key string) (data V, found bool) { - ret, err := c.backend.Get(context.Background(), key).Result() + ret, err := c.backend.Get(context.Background(), c.fullKey(key)).Result() if err != nil { var emptyValue V return emptyValue, false } - switch any(data).(type) { - case string: - return any(ret).(V), true - default: - return any(ret).(V), true - } + return c.toV(ret), true } -// Purge clears the cache completely. +// Purge clears the cache completely. Without RedisKeyPrefix set it flushes the whole redis database. func (c *RedisCache[V]) Purge() { - c.backend.FlushDB(context.Background()) - + if c.redisKeyPrefix == "" { + c.backend.FlushDB(context.Background()) + return + } + // deleted one by one, a multi-key Del is routed by the first key's slot + // and fails across slots on a cluster client + for _, key := range c.scanKeys() { + c.backend.Del(context.Background(), key) + } } // Delete cache item by key func (c *RedisCache[V]) Delete(key string) { - c.backend.Del(context.Background(), key) + c.backend.Del(context.Background(), c.fullKey(key)) } // Keys gets all keys for the cache func (c *RedisCache[V]) Keys() (res []string) { - return c.backend.Keys(context.Background(), "*").Val() + keys := c.scanKeys() + if c.redisKeyPrefix == "" { + return keys + } + res = make([]string, 0, len(keys)) + for _, key := range keys { + res = append(res, strings.TrimPrefix(key, c.redisKeyPrefix)) + } + return res } // Stat returns cache statistics @@ -165,20 +204,30 @@ func (c *RedisCache[V]) size() int64 { } func (c *RedisCache[V]) keys() int { - return int(c.backend.DBSize(context.Background()).Val()) + if c.redisKeyPrefix == "" { + return int(c.backend.DBSize(context.Background()).Val()) + } + return len(c.scanKeys()) } func (c *RedisCache[V]) allowed(key string, data V) bool { - if c.maxKeys > 0 && c.backend.DBSize(context.Background()).Val() >= int64(c.maxKeys) { + if c.maxKeys > 0 && c.keys() >= c.maxKeys { return false } if c.maxKeySize > 0 && len(key) > c.maxKeySize { return false } - if s, ok := any(data).(Sizer); ok { - if c.maxValueSize > 0 && (s.Size() >= c.maxValueSize) { + if size, ok := sizeOf(data); ok { + if c.maxValueSize > 0 && size >= c.maxValueSize { return false } } return true } + +// escapeGlob escapes redis glob-style pattern metacharacters, so a key prefix +// containing them still matches literally +func escapeGlob(s string) string { + replacer := strings.NewReplacer(`\`, `\\`, "*", `\*`, "?", `\?`, "[", `\[`, "]", `\]`) + return replacer.Replace(s) +} diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/scache.go b/backend/vendor/github.com/go-pkgz/lcw/v2/scache.go index d60cfc3e..5fb0124a 100644 --- a/backend/vendor/github.com/go-pkgz/lcw/v2/scache.go +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/scache.go @@ -2,6 +2,7 @@ package lcw import ( "fmt" + "slices" "strings" ) @@ -35,24 +36,24 @@ func (m *Scache[V]) Close() error { return m.lc.Close() } -// Flush clears cache and calls postFlushFn async +// Flush clears keys of the requested partition, matching the requested scopes. +// With no scopes set every key of the partition is removed, keys of other partitions are kept. func (m *Scache[V]) Flush(req FlusherRequest) { - if len(req.scopes) == 0 { - m.lc.Purge() - return - } - - // check if fullKey has matching scopes + // check if fullKey belongs to the requested partition and has matching scopes inScope := func(fullKey string) bool { key, err := parseKey(fullKey) if err != nil { return false } + if key.partition != req.partition { + return false + } + if len(req.scopes) == 0 { // no scopes means the whole partition + return true + } for _, s := range req.scopes { - for _, ks := range key.scopes { - if ks == s { - return true - } + if slices.Contains(key.scopes, s) { + return true } } return false diff --git a/backend/vendor/github.com/go-pkgz/lcw/v2/url.go b/backend/vendor/github.com/go-pkgz/lcw/v2/url.go index 8328696e..a407ce71 100644 --- a/backend/vendor/github.com/go-pkgz/lcw/v2/url.go +++ b/backend/vendor/github.com/go-pkgz/lcw/v2/url.go @@ -1,18 +1,18 @@ package lcw import ( + "errors" "fmt" "net/url" "strconv" "time" - "github.com/hashicorp/go-multierror" "github.com/redis/go-redis/v9" ) // New parses uri and makes any of supported caches // supported URIs: -// - redis://:?db=123&max_keys=10 +// - redis://:?db=123&max_keys=10&redis_key_prefix=lcw: // - mem://lru?max_keys=10&max_cache_size=1024 // - mem://expirable?ttl=30s&max_val_size=100 // - nop:// @@ -55,13 +55,13 @@ func New[V any](uri string) (LoadingCache[V], error) { } func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) { - errs := new(multierror.Error) + var errs []error o := NewOpts[V]() if v := q.Get("max_val_size"); v != "" { vv, e := strconv.Atoi(v) if e != nil { - errs = multierror.Append(errs, fmt.Errorf("max_val_size query param %s: %w", v, e)) + errs = append(errs, fmt.Errorf("max_val_size query param %s: %w", v, e)) } else { opts = append(opts, o.MaxValSize(vv)) } @@ -70,7 +70,7 @@ func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) { if v := q.Get("max_key_size"); v != "" { vv, e := strconv.Atoi(v) if e != nil { - errs = multierror.Append(errs, fmt.Errorf("max_key_size query param %s: %w", v, e)) + errs = append(errs, fmt.Errorf("max_key_size query param %s: %w", v, e)) } else { opts = append(opts, o.MaxKeySize(vv)) } @@ -79,7 +79,7 @@ func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) { if v := q.Get("max_keys"); v != "" { vv, e := strconv.Atoi(v) if e != nil { - errs = multierror.Append(errs, fmt.Errorf("max_keys query param %s: %w", v, e)) + errs = append(errs, fmt.Errorf("max_keys query param %s: %w", v, e)) } else { opts = append(opts, o.MaxKeys(vv)) } @@ -88,7 +88,7 @@ func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) { if v := q.Get("max_cache_size"); v != "" { vv, e := strconv.ParseInt(v, 10, 64) if e != nil { - errs = multierror.Append(errs, fmt.Errorf("max_cache_size query param %s: %w", v, e)) + errs = append(errs, fmt.Errorf("max_cache_size query param %s: %w", v, e)) } else { opts = append(opts, o.MaxCacheSize(vv)) } @@ -97,13 +97,17 @@ func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) { if v := q.Get("ttl"); v != "" { vv, e := time.ParseDuration(v) if e != nil { - errs = multierror.Append(errs, fmt.Errorf("ttl query param %s: %w", v, e)) + errs = append(errs, fmt.Errorf("ttl query param %s: %w", v, e)) } else { opts = append(opts, o.TTL(vv)) } } - return opts, errs.ErrorOrNil() + if v := q.Get("redis_key_prefix"); v != "" { + opts = append(opts, o.RedisKeyPrefix(v)) + } + + return opts, errors.Join(errs...) } func redisOptionsFromURL(u *url.URL) (*redis.Options, error) { diff --git a/backend/vendor/github.com/go-pkgz/routegroup/.golangci.yml b/backend/vendor/github.com/go-pkgz/routegroup/.golangci.yml index f8002ab0..c9171a4b 100644 --- a/backend/vendor/github.com/go-pkgz/routegroup/.golangci.yml +++ b/backend/vendor/github.com/go-pkgz/routegroup/.golangci.yml @@ -44,6 +44,10 @@ linters: - linters: - gosec text: 'G114: Use of net/http serve function that has no support for setting timeouts' + - linters: + - gosec + path: _test\.go$ + text: 'G705: XSS via taint analysis' - linters: - revive - unparam diff --git a/backend/vendor/github.com/go-pkgz/routegroup/README.md b/backend/vendor/github.com/go-pkgz/routegroup/README.md index 174d712b..1fd24d98 100644 --- a/backend/vendor/github.com/go-pkgz/routegroup/README.md +++ b/backend/vendor/github.com/go-pkgz/routegroup/README.md @@ -6,7 +6,7 @@ ## Features - Simple and intuitive API for route grouping and route mounting. -- Lightweight, just about 100 LOC +- Lightweight, contained in a single file - Easy middleware integration for individual routes or groups of routes. - Seamless integration with Go's standard `http.ServeMux`. - Fully compatible with the `http.Handler` interface and can be used as a drop-in replacement for `http.ServeMux`. @@ -133,7 +133,7 @@ router.Group().Route(func(b *routegroup.Bundle) { **Setting optional `NotFoundHandler`** -It is possible to set a custom `NotFoundHandler` for the group. This handler will be called when no other route matches the request: +Set a custom `NotFoundHandler` for the group. It runs only when `http.ServeMux` would return 404: ```go group.NotFoundHandler(func(w http.ResponseWriter, _ *http.Request) { @@ -141,11 +141,11 @@ group.NotFoundHandler(func(w http.ResponseWriter, _ *http.Request) { } ``` -If a custom `NotFoundHandler` is not configured, `routegroup` will default to using the standard library behavior. +Without a custom `NotFoundHandler`, `routegroup` uses the standard library behavior. -Note on 405: In the current design, `routegroup` applies root-level middlewares to all requests at the top level without installing a catch‑all route. This preserves native `405 Method Not Allowed` responses from `http.ServeMux` when a path exists but a wrong method is used. A configured `NotFoundHandler` is only invoked when no route matches; it does not interfere with 405 handling. The custom `NotFoundHandler` will have the root bundle's global middlewares applied to it. +Native mux responses: `routegroup` applies root-level middlewares to all requests without installing a catch‑all route. A configured `NotFoundHandler` runs only when `http.ServeMux` would return 404. Method mismatches return `405 Method Not Allowed` with an `Allow` header, and `http.ServeMux` performs path-cleanup redirects. Root-level middlewares apply to the custom handler. -Legacy note: `DisableNotFoundHandler()` is now a no‑op and preserved only for API compatibility. +`DisableNotFoundHandler()` has no effect. ### Middleware Ordering @@ -289,7 +289,7 @@ http.ListenAndServe(":8080", mux) - Wrong method on an existing path returns `405 Method Not Allowed` (with an `Allow` header). - Unknown path returns `404 Not Found`. -You can optionally configure a custom 404 handler with `NotFoundHandler(fn)`. It will run only when no route matches and does not affect 405 handling. The custom handler will have global middlewares applied to it. The legacy `DisableNotFoundHandler()` is now a no‑op and kept only for compatibility. +You can optionally configure a custom 404 handler with `NotFoundHandler(fn)`. It runs only when `http.ServeMux` would return 404 and does not affect method mismatch responses or path-cleanup redirects. Global middlewares apply to the custom handler. `DisableNotFoundHandler()` has no effect. ### HandleFiles helper diff --git a/backend/vendor/github.com/go-pkgz/routegroup/group.go b/backend/vendor/github.com/go-pkgz/routegroup/group.go index bc49932a..ad40c04d 100644 --- a/backend/vendor/github.com/go-pkgz/routegroup/group.go +++ b/backend/vendor/github.com/go-pkgz/routegroup/group.go @@ -63,14 +63,19 @@ func (b *Bundle) ServeHTTP(w http.ResponseWriter, r *http.Request) { // but intercept 404s to use custom handler if provided muxHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if pattern == "" && root.notFound != nil { - // no route matched, need to check if it's a true 404 or a 405 - // probe the mux to see what status it would return - probe := &statusRecorder{status: http.StatusOK} - b.mux.ServeHTTP(probe, r) + handler, currentPattern := b.mux.Handler(r) + if currentPattern != "" { + b.mux.ServeHTTP(w, r) + return + } - // if mux wants to return 405 (Method Not Allowed), let it handle the request - // to preserve the proper 405 response and Allow header - if probe.status == http.StatusMethodNotAllowed { + // no route matched, need to check if it's a true 404 + // probe the synthetic handler to see what status it would return + probe := &statusRecorder{status: http.StatusOK} + handler.ServeHTTP(probe, r) + + // let the mux handle redirects and method mismatches + if probe.status != http.StatusNotFound { b.mux.ServeHTTP(w, r) return } @@ -181,13 +186,13 @@ func (b *Bundle) Handler(r *http.Request) (h http.Handler, pattern string) { return b.mux.Handler(r) } -// DisableNotFoundHandler used to disable auto-registration of a catch-all 404. -// Deprecated: now a no-op retained for API compatibility. +// DisableNotFoundHandler has no effect. +// +// Deprecated: this method has no effect. func (b *Bundle) DisableNotFoundHandler() {} -// NotFoundHandler sets a custom handler for any unmatched routes (404 responses). -// Note: This handler is only used for true 404s. Requests to valid paths with -// incorrect HTTP methods will still return 405 Method Not Allowed with Allow header. +// NotFoundHandler sets a custom handler for true 404 responses. +// Method mismatches and path-cleanup redirects are handled by http.ServeMux. func (b *Bundle) NotFoundHandler(handler http.HandlerFunc) { // always set on the root bundle so custom 404 works regardless of which bundle serves. if b.root != nil { diff --git a/backend/vendor/github.com/go-pkgz/syncs/.golangci.yml b/backend/vendor/github.com/go-pkgz/syncs/.golangci.yml index 192d484a..6201377a 100644 --- a/backend/vendor/github.com/go-pkgz/syncs/.golangci.yml +++ b/backend/vendor/github.com/go-pkgz/syncs/.golangci.yml @@ -1,83 +1,92 @@ -linters-settings: - govet: - check-shadowing: true - golint: - min-confidence: 0.6 - gocyclo: - min-complexity: 15 - maligned: - suggest-new: true - dupl: - threshold: 100 - goconst: - min-len: 2 - min-occurrences: 2 - misspell: - locale: US - lll: - line-length: 140 - gocritic: - enabled-tags: - - performance - - style - - experimental - disabled-checks: - - wrapperFunc - - hugeParam - - rangeValCopy - -linters: - disable-all: true - enable: - - megacheck - - revive - - govet - - unconvert - - gas - - misspell - - unused - - typecheck - - ineffassign - - stylecheck - - gochecknoinits - - exportloopref - - nakedret - - gosimple - - prealloc - - fast: false - - +version: "2" run: - # modules-download-mode: vendor - skip-dirs: - - vendor concurrency: 4 +linters: + default: none + enable: + - gochecknoinits + - gocritic + - gosec + - govet + - ineffassign + - nakedret + - prealloc + - revive + - staticcheck + - unconvert + - unparam + - unused + settings: + goconst: + min-len: 2 + min-occurrences: 2 + gocritic: + disabled-checks: + - wrapperFunc + - hugeParam + - rangeValCopy + enabled-tags: + - performance + - style + - experimental + gocyclo: + min-complexity: 15 + govet: + enable-all: true + disable: + - fieldalignment + lll: + line-length: 140 + misspell: + locale: US -issues: - exclude-rules: - - text: "should have a package comment, unless it's in another file for this package" - linters: - - golint - - text: "exitAfterDefer:" - linters: - - gocritic - - text: "whyNoLint: include an explanation for nolint directive" - linters: - - gocritic - - text: "go.mongodb.org/mongo-driver/bson/primitive.E" - linters: - - govet - - text: "weak cryptographic primitive" - linters: - - gosec - - text: "at least one file in a package should have a package comment" - linters: - - stylecheck - - text: "should have a package comment" - linters: - - revive - - text: 'Deferring unsafe method "Close" on type "io.ReadCloser"' - linters: - - gosec - exclude-use-default: false + exclusions: + generated: lax + rules: + - linters: + - gocritic + text: "exitAfterDefer:" + - linters: + - gocritic + text: "whyNoLint: include an explanation for nolint directive" + - linters: + - govet + text: "go.mongodb.org/mongo-driver/bson/primitive.E" + - linters: + - gosec + text: "weak cryptographic primitive" + - linters: + - gosec + text: "integer overflow conversion" + - linters: + - revive + text: "should have a package comment" + - linters: + - staticcheck + text: "at least one file in a package should have a package comment" + - linters: + - gocritic + text: "commentedOutCode: may want to remove commented-out code" + - linters: + - gocritic + text: "unnamedResult: consider giving a name to these results" + - linters: + - revive + text: "var-naming: don't use an underscore in package name" + - linters: + - staticcheck + text: "should not use underscores in package names" + - linters: + - govet + text: "struct literal uses unkeyed fields" + - linters: + - unparam + - unused + - revive + path: _test\.go$ + text: "unused-parameter" + paths: + - vendor + - third_party$ + - builtin$ + - examples$ diff --git a/backend/vendor/github.com/go-pkgz/syncs/README.md b/backend/vendor/github.com/go-pkgz/syncs/README.md index 1ce406f5..f896c2de 100644 --- a/backend/vendor/github.com/go-pkgz/syncs/README.md +++ b/backend/vendor/github.com/go-pkgz/syncs/README.md @@ -72,7 +72,7 @@ It can work as a regular errgrp.Group or with early termination. It is thread-sa ```go ewg := syncs.NewErrSizedGroup(5, syncs.Preemptive) // error wait group with max size=5, don't try to start more if any error happened for i :=0; i<10; i++ { - ewg.Go(func(ctx context.Context) error { // Go here could be blocked if trying to run >5 at the same time + ewg.Go(func() error { // Go here could be blocked if trying to run >5 at the same time err := doThings(ctx) // only 5 of these will run in parallel return err }) @@ -80,3 +80,11 @@ It can work as a regular errgrp.Group or with early termination. It is thread-sa err := ewg.Wait() ``` +`Wait` returns all the collected errors as `*MultiError`, which implements `Unwrap() []error`, so `errors.Is` and `errors.As` match any of them: + +```go + if err := ewg.Wait(); errors.Is(err, context.Canceled) { + // at least one of the goroutines was canceled + } +``` + diff --git a/backend/vendor/github.com/go-pkgz/syncs/errsizedgroup.go b/backend/vendor/github.com/go-pkgz/syncs/errsizedgroup.go index d149ce67..27b1d7b2 100644 --- a/backend/vendor/github.com/go-pkgz/syncs/errsizedgroup.go +++ b/backend/vendor/github.com/go-pkgz/syncs/errsizedgroup.go @@ -2,6 +2,7 @@ package syncs import ( "fmt" + "slices" "strings" "sync" ) @@ -15,7 +16,6 @@ type ErrSizedGroup struct { sema Locker err *MultiError - errLock sync.RWMutex errOnce sync.Once } @@ -88,8 +88,6 @@ func (g *ErrSizedGroup) Go(f func() error) { if !g.termOnError { return false } - g.errLock.RLock() - defer g.errLock.RUnlock() return g.err.ErrorOrNil() != nil } @@ -109,9 +107,7 @@ func (g *ErrSizedGroup) Go(f func() error) { } if err := f(); err != nil { - g.errLock.Lock() - g.err = g.err.append(err) - g.errLock.Unlock() + g.err.append(err) } }() } @@ -129,11 +125,10 @@ type MultiError struct { lock sync.Mutex } -func (m *MultiError) append(err error) *MultiError { +func (m *MultiError) append(err error) { m.lock.Lock() m.errors = append(m.errors, err) m.lock.Unlock() - return m } // ErrorOrNil returns nil if no errors or multierror if errors occurred @@ -162,8 +157,16 @@ func (m *MultiError) Error() string { return fmt.Sprintf("%d error(s) occurred: %s", len(m.errors), strings.Join(errs, ", ")) } +// Errors returns all errors collected func (m *MultiError) Errors() []error { m.lock.Lock() defer m.lock.Unlock() - return m.errors + return slices.Clone(m.errors) +} + +// Unwrap returns all errors collected, allows errors.Is and errors.As to match any of them +func (m *MultiError) Unwrap() []error { + m.lock.Lock() + defer m.lock.Unlock() + return slices.Clone(m.errors) } diff --git a/backend/vendor/github.com/go-pkgz/syncs/semaphore.go b/backend/vendor/github.com/go-pkgz/syncs/semaphore.go index e99254a2..c9b56b5c 100644 --- a/backend/vendor/github.com/go-pkgz/syncs/semaphore.go +++ b/backend/vendor/github.com/go-pkgz/syncs/semaphore.go @@ -8,9 +8,8 @@ type Locker interface { TryLock() bool } -// Semaphore implementation, counted lock only. Implements sync.Locker interface, thread safe. +// Semaphore implementation, counted lock only. Implements Locker interface, thread safe. type semaphore struct { - Locker ch chan struct{} } diff --git a/backend/vendor/github.com/go-pkgz/syncs/sizedgroup.go b/backend/vendor/github.com/go-pkgz/syncs/sizedgroup.go index b73f77a4..be3347d4 100644 --- a/backend/vendor/github.com/go-pkgz/syncs/sizedgroup.go +++ b/backend/vendor/github.com/go-pkgz/syncs/sizedgroup.go @@ -17,7 +17,7 @@ type SizedGroup struct { // NewSizedGroup makes wait group with limited size alive goroutines func NewSizedGroup(size int, opts ...GroupOption) *SizedGroup { res := SizedGroup{sema: NewSemaphore(size)} - res.options.ctx = context.Background() + res.ctx = context.Background() for _, opt := range opts { opt(&res.options) } diff --git a/backend/vendor/github.com/hashicorp/errwrap/LICENSE b/backend/vendor/github.com/hashicorp/errwrap/LICENSE deleted file mode 100644 index c33dcc7c..00000000 --- a/backend/vendor/github.com/hashicorp/errwrap/LICENSE +++ /dev/null @@ -1,354 +0,0 @@ -Mozilla Public License, version 2.0 - -1. Definitions - -1.1. “Contributor” - - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. - -1.2. “Contributor Version” - - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” - - means Covered Software of a particular Contributor. - -1.4. “Covered Software” - - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. - -1.5. “Incompatible With Secondary Licenses” - means - - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or - - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. - -1.6. “Executable Form” - - means any form of the work other than Source Code Form. - -1.7. “Larger Work” - - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. - -1.8. “License” - - means this document. - -1.9. “Licensable” - - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. - -1.10. “Modifications” - - means any of the following: - - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or - - b. any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor - - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. - -1.12. “Secondary License” - - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. - -1.13. “Source Code Form” - - means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) - - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. - - -2. License Grants and Conditions - -2.1. Grants - - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: - - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and - - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. - -2.2. Effective Date - - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. - -2.3. Limitations on Grant Scope - - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: - - a. for any code that a Contributor has removed from Covered Software; or - - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. - - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). - -2.4. Subsequent Licenses - - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). - -2.5. Representation - - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. - -2.6. Fair Use - - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. - - -3. Responsibilities - -3.1. Distribution of Source Form - - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - - If You distribute Covered Software in Executable Form then: - - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and - - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). - -3.4. Notices - - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. - -6. Disclaimer of Warranty - - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. - -7. Limitation of Liability - - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. - -8. Litigation - - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. - - -10. Versions of the License - -10.1. New Versions - - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. - -10.2. Effect of New Versions - - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. - -10.3. Modified Versions - - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. - diff --git a/backend/vendor/github.com/hashicorp/errwrap/README.md b/backend/vendor/github.com/hashicorp/errwrap/README.md deleted file mode 100644 index 444df08f..00000000 --- a/backend/vendor/github.com/hashicorp/errwrap/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# errwrap - -`errwrap` is a package for Go that formalizes the pattern of wrapping errors -and checking if an error contains another error. - -There is a common pattern in Go of taking a returned `error` value and -then wrapping it (such as with `fmt.Errorf`) before returning it. The problem -with this pattern is that you completely lose the original `error` structure. - -Arguably the _correct_ approach is that you should make a custom structure -implementing the `error` interface, and have the original error as a field -on that structure, such [as this example](http://golang.org/pkg/os/#PathError). -This is a good approach, but you have to know the entire chain of possible -rewrapping that happens, when you might just care about one. - -`errwrap` formalizes this pattern (it doesn't matter what approach you use -above) by giving a single interface for wrapping errors, checking if a specific -error is wrapped, and extracting that error. - -## Installation and Docs - -Install using `go get github.com/hashicorp/errwrap`. - -Full documentation is available at -http://godoc.org/github.com/hashicorp/errwrap - -## Usage - -#### Basic Usage - -Below is a very basic example of its usage: - -```go -// A function that always returns an error, but wraps it, like a real -// function might. -func tryOpen() error { - _, err := os.Open("/i/dont/exist") - if err != nil { - return errwrap.Wrapf("Doesn't exist: {{err}}", err) - } - - return nil -} - -func main() { - err := tryOpen() - - // We can use the Contains helpers to check if an error contains - // another error. It is safe to do this with a nil error, or with - // an error that doesn't even use the errwrap package. - if errwrap.Contains(err, "does not exist") { - // Do something - } - if errwrap.ContainsType(err, new(os.PathError)) { - // Do something - } - - // Or we can use the associated `Get` functions to just extract - // a specific error. This would return nil if that specific error doesn't - // exist. - perr := errwrap.GetType(err, new(os.PathError)) -} -``` - -#### Custom Types - -If you're already making custom types that properly wrap errors, then -you can get all the functionality of `errwraps.Contains` and such by -implementing the `Wrapper` interface with just one function. Example: - -```go -type AppError { - Code ErrorCode - Err error -} - -func (e *AppError) WrappedErrors() []error { - return []error{e.Err} -} -``` - -Now this works: - -```go -err := &AppError{Err: fmt.Errorf("an error")} -if errwrap.ContainsType(err, fmt.Errorf("")) { - // This will work! -} -``` diff --git a/backend/vendor/github.com/hashicorp/errwrap/errwrap.go b/backend/vendor/github.com/hashicorp/errwrap/errwrap.go deleted file mode 100644 index 44e368e5..00000000 --- a/backend/vendor/github.com/hashicorp/errwrap/errwrap.go +++ /dev/null @@ -1,178 +0,0 @@ -// Package errwrap implements methods to formalize error wrapping in Go. -// -// All of the top-level functions that take an `error` are built to be able -// to take any error, not just wrapped errors. This allows you to use errwrap -// without having to type-check and type-cast everywhere. -package errwrap - -import ( - "errors" - "reflect" - "strings" -) - -// WalkFunc is the callback called for Walk. -type WalkFunc func(error) - -// Wrapper is an interface that can be implemented by custom types to -// have all the Contains, Get, etc. functions in errwrap work. -// -// When Walk reaches a Wrapper, it will call the callback for every -// wrapped error in addition to the wrapper itself. Since all the top-level -// functions in errwrap use Walk, this means that all those functions work -// with your custom type. -type Wrapper interface { - WrappedErrors() []error -} - -// Wrap defines that outer wraps inner, returning an error type that -// can be cleanly used with the other methods in this package, such as -// Contains, GetAll, etc. -// -// This function won't modify the error message at all (the outer message -// will be used). -func Wrap(outer, inner error) error { - return &wrappedError{ - Outer: outer, - Inner: inner, - } -} - -// Wrapf wraps an error with a formatting message. This is similar to using -// `fmt.Errorf` to wrap an error. If you're using `fmt.Errorf` to wrap -// errors, you should replace it with this. -// -// format is the format of the error message. The string '{{err}}' will -// be replaced with the original error message. -// -// Deprecated: Use fmt.Errorf() -func Wrapf(format string, err error) error { - outerMsg := "" - if err != nil { - outerMsg = err.Error() - } - - outer := errors.New(strings.Replace( - format, "{{err}}", outerMsg, -1)) - - return Wrap(outer, err) -} - -// Contains checks if the given error contains an error with the -// message msg. If err is not a wrapped error, this will always return -// false unless the error itself happens to match this msg. -func Contains(err error, msg string) bool { - return len(GetAll(err, msg)) > 0 -} - -// ContainsType checks if the given error contains an error with -// the same concrete type as v. If err is not a wrapped error, this will -// check the err itself. -func ContainsType(err error, v interface{}) bool { - return len(GetAllType(err, v)) > 0 -} - -// Get is the same as GetAll but returns the deepest matching error. -func Get(err error, msg string) error { - es := GetAll(err, msg) - if len(es) > 0 { - return es[len(es)-1] - } - - return nil -} - -// GetType is the same as GetAllType but returns the deepest matching error. -func GetType(err error, v interface{}) error { - es := GetAllType(err, v) - if len(es) > 0 { - return es[len(es)-1] - } - - return nil -} - -// GetAll gets all the errors that might be wrapped in err with the -// given message. The order of the errors is such that the outermost -// matching error (the most recent wrap) is index zero, and so on. -func GetAll(err error, msg string) []error { - var result []error - - Walk(err, func(err error) { - if err.Error() == msg { - result = append(result, err) - } - }) - - return result -} - -// GetAllType gets all the errors that are the same type as v. -// -// The order of the return value is the same as described in GetAll. -func GetAllType(err error, v interface{}) []error { - var result []error - - var search string - if v != nil { - search = reflect.TypeOf(v).String() - } - Walk(err, func(err error) { - var needle string - if err != nil { - needle = reflect.TypeOf(err).String() - } - - if needle == search { - result = append(result, err) - } - }) - - return result -} - -// Walk walks all the wrapped errors in err and calls the callback. If -// err isn't a wrapped error, this will be called once for err. If err -// is a wrapped error, the callback will be called for both the wrapper -// that implements error as well as the wrapped error itself. -func Walk(err error, cb WalkFunc) { - if err == nil { - return - } - - switch e := err.(type) { - case *wrappedError: - cb(e.Outer) - Walk(e.Inner, cb) - case Wrapper: - cb(err) - - for _, err := range e.WrappedErrors() { - Walk(err, cb) - } - case interface{ Unwrap() error }: - cb(err) - Walk(e.Unwrap(), cb) - default: - cb(err) - } -} - -// wrappedError is an implementation of error that has both the -// outer and inner errors. -type wrappedError struct { - Outer error - Inner error -} - -func (w *wrappedError) Error() string { - return w.Outer.Error() -} - -func (w *wrappedError) WrappedErrors() []error { - return []error{w.Outer, w.Inner} -} - -func (w *wrappedError) Unwrap() error { - return w.Inner -} diff --git a/backend/vendor/github.com/hashicorp/go-multierror/LICENSE b/backend/vendor/github.com/hashicorp/go-multierror/LICENSE deleted file mode 100644 index 82b4de97..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/LICENSE +++ /dev/null @@ -1,353 +0,0 @@ -Mozilla Public License, version 2.0 - -1. Definitions - -1.1. “Contributor” - - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. - -1.2. “Contributor Version” - - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” - - means Covered Software of a particular Contributor. - -1.4. “Covered Software” - - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. - -1.5. “Incompatible With Secondary Licenses” - means - - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or - - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. - -1.6. “Executable Form” - - means any form of the work other than Source Code Form. - -1.7. “Larger Work” - - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. - -1.8. “License” - - means this document. - -1.9. “Licensable” - - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. - -1.10. “Modifications” - - means any of the following: - - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or - - b. any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor - - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. - -1.12. “Secondary License” - - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. - -1.13. “Source Code Form” - - means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) - - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. - - -2. License Grants and Conditions - -2.1. Grants - - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: - - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and - - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. - -2.2. Effective Date - - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. - -2.3. Limitations on Grant Scope - - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: - - a. for any code that a Contributor has removed from Covered Software; or - - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. - - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). - -2.4. Subsequent Licenses - - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). - -2.5. Representation - - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. - -2.6. Fair Use - - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. - - -3. Responsibilities - -3.1. Distribution of Source Form - - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - - If You distribute Covered Software in Executable Form then: - - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and - - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). - -3.4. Notices - - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. - -6. Disclaimer of Warranty - - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. - -7. Limitation of Liability - - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. - -8. Litigation - - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. - - -10. Versions of the License - -10.1. New Versions - - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. - -10.2. Effect of New Versions - - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. - -10.3. Modified Versions - - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. diff --git a/backend/vendor/github.com/hashicorp/go-multierror/Makefile b/backend/vendor/github.com/hashicorp/go-multierror/Makefile deleted file mode 100644 index b97cd6ed..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -TEST?=./... - -default: test - -# test runs the test suite and vets the code. -test: generate - @echo "==> Running tests..." - @go list $(TEST) \ - | grep -v "/vendor/" \ - | xargs -n1 go test -timeout=60s -parallel=10 ${TESTARGS} - -# testrace runs the race checker -testrace: generate - @echo "==> Running tests (race)..." - @go list $(TEST) \ - | grep -v "/vendor/" \ - | xargs -n1 go test -timeout=60s -race ${TESTARGS} - -# updatedeps installs all the dependencies needed to run and build. -updatedeps: - @sh -c "'${CURDIR}/scripts/deps.sh' '${NAME}'" - -# generate runs `go generate` to build the dynamically generated source files. -generate: - @echo "==> Generating..." - @find . -type f -name '.DS_Store' -delete - @go list ./... \ - | grep -v "/vendor/" \ - | xargs -n1 go generate - -.PHONY: default test testrace updatedeps generate diff --git a/backend/vendor/github.com/hashicorp/go-multierror/README.md b/backend/vendor/github.com/hashicorp/go-multierror/README.md deleted file mode 100644 index 71dd308e..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# go-multierror - -[![CircleCI](https://img.shields.io/circleci/build/github/hashicorp/go-multierror/master)](https://circleci.com/gh/hashicorp/go-multierror) -[![Go Reference](https://pkg.go.dev/badge/github.com/hashicorp/go-multierror.svg)](https://pkg.go.dev/github.com/hashicorp/go-multierror) -![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/hashicorp/go-multierror) - -[circleci]: https://app.circleci.com/pipelines/github/hashicorp/go-multierror -[godocs]: https://pkg.go.dev/github.com/hashicorp/go-multierror - -`go-multierror` is a package for Go that provides a mechanism for -representing a list of `error` values as a single `error`. - -This allows a function in Go to return an `error` that might actually -be a list of errors. If the caller knows this, they can unwrap the -list and access the errors. If the caller doesn't know, the error -formats to a nice human-readable format. - -`go-multierror` is fully compatible with the Go standard library -[errors](https://golang.org/pkg/errors/) package, including the -functions `As`, `Is`, and `Unwrap`. This provides a standardized approach -for introspecting on error values. - -## Installation and Docs - -Install using `go get github.com/hashicorp/go-multierror`. - -Full documentation is available at -https://pkg.go.dev/github.com/hashicorp/go-multierror - -### Requires go version 1.13 or newer - -`go-multierror` requires go version 1.13 or newer. Go 1.13 introduced -[error wrapping](https://golang.org/doc/go1.13#error_wrapping), which -this library takes advantage of. - -If you need to use an earlier version of go, you can use the -[v1.0.0](https://github.com/hashicorp/go-multierror/tree/v1.0.0) -tag, which doesn't rely on features in go 1.13. - -If you see compile errors that look like the below, it's likely that -you're on an older version of go: - -``` -/go/src/github.com/hashicorp/go-multierror/multierror.go:112:9: undefined: errors.As -/go/src/github.com/hashicorp/go-multierror/multierror.go:117:9: undefined: errors.Is -``` - -## Usage - -go-multierror is easy to use and purposely built to be unobtrusive in -existing Go applications/libraries that may not be aware of it. - -**Building a list of errors** - -The `Append` function is used to create a list of errors. This function -behaves a lot like the Go built-in `append` function: it doesn't matter -if the first argument is nil, a `multierror.Error`, or any other `error`, -the function behaves as you would expect. - -```go -var result error - -if err := step1(); err != nil { - result = multierror.Append(result, err) -} -if err := step2(); err != nil { - result = multierror.Append(result, err) -} - -return result -``` - -**Customizing the formatting of the errors** - -By specifying a custom `ErrorFormat`, you can customize the format -of the `Error() string` function: - -```go -var result *multierror.Error - -// ... accumulate errors here, maybe using Append - -if result != nil { - result.ErrorFormat = func([]error) string { - return "errors!" - } -} -``` - -**Accessing the list of errors** - -`multierror.Error` implements `error` so if the caller doesn't know about -multierror, it will work just fine. But if you're aware a multierror might -be returned, you can use type switches to access the list of errors: - -```go -if err := something(); err != nil { - if merr, ok := err.(*multierror.Error); ok { - // Use merr.Errors - } -} -``` - -You can also use the standard [`errors.Unwrap`](https://golang.org/pkg/errors/#Unwrap) -function. This will continue to unwrap into subsequent errors until none exist. - -**Extracting an error** - -The standard library [`errors.As`](https://golang.org/pkg/errors/#As) -function can be used directly with a multierror to extract a specific error: - -```go -// Assume err is a multierror value -err := somefunc() - -// We want to know if "err" has a "RichErrorType" in it and extract it. -var errRich RichErrorType -if errors.As(err, &errRich) { - // It has it, and now errRich is populated. -} -``` - -**Checking for an exact error value** - -Some errors are returned as exact errors such as the [`ErrNotExist`](https://golang.org/pkg/os/#pkg-variables) -error in the `os` package. You can check if this error is present by using -the standard [`errors.Is`](https://golang.org/pkg/errors/#Is) function. - -```go -// Assume err is a multierror value -err := somefunc() -if errors.Is(err, os.ErrNotExist) { - // err contains os.ErrNotExist -} -``` - -**Returning a multierror only if there are errors** - -If you build a `multierror.Error`, you can use the `ErrorOrNil` function -to return an `error` implementation only if there are errors to return: - -```go -var result *multierror.Error - -// ... accumulate errors here - -// Return the `error` only if errors were added to the multierror, otherwise -// return nil since there are no errors. -return result.ErrorOrNil() -``` diff --git a/backend/vendor/github.com/hashicorp/go-multierror/append.go b/backend/vendor/github.com/hashicorp/go-multierror/append.go deleted file mode 100644 index 3e2589bf..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/append.go +++ /dev/null @@ -1,43 +0,0 @@ -package multierror - -// Append is a helper function that will append more errors -// onto an Error in order to create a larger multi-error. -// -// If err is not a multierror.Error, then it will be turned into -// one. If any of the errs are multierr.Error, they will be flattened -// one level into err. -// Any nil errors within errs will be ignored. If err is nil, a new -// *Error will be returned. -func Append(err error, errs ...error) *Error { - switch err := err.(type) { - case *Error: - // Typed nils can reach here, so initialize if we are nil - if err == nil { - err = new(Error) - } - - // Go through each error and flatten - for _, e := range errs { - switch e := e.(type) { - case *Error: - if e != nil { - err.Errors = append(err.Errors, e.Errors...) - } - default: - if e != nil { - err.Errors = append(err.Errors, e) - } - } - } - - return err - default: - newErrs := make([]error, 0, len(errs)+1) - if err != nil { - newErrs = append(newErrs, err) - } - newErrs = append(newErrs, errs...) - - return Append(&Error{}, newErrs...) - } -} diff --git a/backend/vendor/github.com/hashicorp/go-multierror/flatten.go b/backend/vendor/github.com/hashicorp/go-multierror/flatten.go deleted file mode 100644 index aab8e9ab..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/flatten.go +++ /dev/null @@ -1,26 +0,0 @@ -package multierror - -// Flatten flattens the given error, merging any *Errors together into -// a single *Error. -func Flatten(err error) error { - // If it isn't an *Error, just return the error as-is - if _, ok := err.(*Error); !ok { - return err - } - - // Otherwise, make the result and flatten away! - flatErr := new(Error) - flatten(err, flatErr) - return flatErr -} - -func flatten(err error, flatErr *Error) { - switch err := err.(type) { - case *Error: - for _, e := range err.Errors { - flatten(e, flatErr) - } - default: - flatErr.Errors = append(flatErr.Errors, err) - } -} diff --git a/backend/vendor/github.com/hashicorp/go-multierror/format.go b/backend/vendor/github.com/hashicorp/go-multierror/format.go deleted file mode 100644 index 47f13c49..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/format.go +++ /dev/null @@ -1,27 +0,0 @@ -package multierror - -import ( - "fmt" - "strings" -) - -// ErrorFormatFunc is a function callback that is called by Error to -// turn the list of errors into a string. -type ErrorFormatFunc func([]error) string - -// ListFormatFunc is a basic formatter that outputs the number of errors -// that occurred along with a bullet point list of the errors. -func ListFormatFunc(es []error) string { - if len(es) == 1 { - return fmt.Sprintf("1 error occurred:\n\t* %s\n\n", es[0]) - } - - points := make([]string, len(es)) - for i, err := range es { - points[i] = fmt.Sprintf("* %s", err) - } - - return fmt.Sprintf( - "%d errors occurred:\n\t%s\n\n", - len(es), strings.Join(points, "\n\t")) -} diff --git a/backend/vendor/github.com/hashicorp/go-multierror/group.go b/backend/vendor/github.com/hashicorp/go-multierror/group.go deleted file mode 100644 index 9c29efb7..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/group.go +++ /dev/null @@ -1,38 +0,0 @@ -package multierror - -import "sync" - -// Group is a collection of goroutines which return errors that need to be -// coalesced. -type Group struct { - mutex sync.Mutex - err *Error - wg sync.WaitGroup -} - -// Go calls the given function in a new goroutine. -// -// If the function returns an error it is added to the group multierror which -// is returned by Wait. -func (g *Group) Go(f func() error) { - g.wg.Add(1) - - go func() { - defer g.wg.Done() - - if err := f(); err != nil { - g.mutex.Lock() - g.err = Append(g.err, err) - g.mutex.Unlock() - } - }() -} - -// Wait blocks until all function calls from the Go method have returned, then -// returns the multierror. -func (g *Group) Wait() *Error { - g.wg.Wait() - g.mutex.Lock() - defer g.mutex.Unlock() - return g.err -} diff --git a/backend/vendor/github.com/hashicorp/go-multierror/multierror.go b/backend/vendor/github.com/hashicorp/go-multierror/multierror.go deleted file mode 100644 index f5457432..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/multierror.go +++ /dev/null @@ -1,121 +0,0 @@ -package multierror - -import ( - "errors" - "fmt" -) - -// Error is an error type to track multiple errors. This is used to -// accumulate errors in cases and return them as a single "error". -type Error struct { - Errors []error - ErrorFormat ErrorFormatFunc -} - -func (e *Error) Error() string { - fn := e.ErrorFormat - if fn == nil { - fn = ListFormatFunc - } - - return fn(e.Errors) -} - -// ErrorOrNil returns an error interface if this Error represents -// a list of errors, or returns nil if the list of errors is empty. This -// function is useful at the end of accumulation to make sure that the value -// returned represents the existence of errors. -func (e *Error) ErrorOrNil() error { - if e == nil { - return nil - } - if len(e.Errors) == 0 { - return nil - } - - return e -} - -func (e *Error) GoString() string { - return fmt.Sprintf("*%#v", *e) -} - -// WrappedErrors returns the list of errors that this Error is wrapping. It is -// an implementation of the errwrap.Wrapper interface so that multierror.Error -// can be used with that library. -// -// This method is not safe to be called concurrently. Unlike accessing the -// Errors field directly, this function also checks if the multierror is nil to -// prevent a null-pointer panic. It satisfies the errwrap.Wrapper interface. -func (e *Error) WrappedErrors() []error { - if e == nil { - return nil - } - return e.Errors -} - -// Unwrap returns an error from Error (or nil if there are no errors). -// This error returned will further support Unwrap to get the next error, -// etc. The order will match the order of Errors in the multierror.Error -// at the time of calling. -// -// The resulting error supports errors.As/Is/Unwrap so you can continue -// to use the stdlib errors package to introspect further. -// -// This will perform a shallow copy of the errors slice. Any errors appended -// to this error after calling Unwrap will not be available until a new -// Unwrap is called on the multierror.Error. -func (e *Error) Unwrap() error { - // If we have no errors then we do nothing - if e == nil || len(e.Errors) == 0 { - return nil - } - - // If we have exactly one error, we can just return that directly. - if len(e.Errors) == 1 { - return e.Errors[0] - } - - // Shallow copy the slice - errs := make([]error, len(e.Errors)) - copy(errs, e.Errors) - return chain(errs) -} - -// chain implements the interfaces necessary for errors.Is/As/Unwrap to -// work in a deterministic way with multierror. A chain tracks a list of -// errors while accounting for the current represented error. This lets -// Is/As be meaningful. -// -// Unwrap returns the next error. In the cleanest form, Unwrap would return -// the wrapped error here but we can't do that if we want to properly -// get access to all the errors. Instead, users are recommended to use -// Is/As to get the correct error type out. -// -// Precondition: []error is non-empty (len > 0) -type chain []error - -// Error implements the error interface -func (e chain) Error() string { - return e[0].Error() -} - -// Unwrap implements errors.Unwrap by returning the next error in the -// chain or nil if there are no more errors. -func (e chain) Unwrap() error { - if len(e) == 1 { - return nil - } - - return e[1:] -} - -// As implements errors.As by attempting to map to the current value. -func (e chain) As(target interface{}) bool { - return errors.As(e[0], target) -} - -// Is implements errors.Is by comparing the current value directly. -func (e chain) Is(target error) bool { - return errors.Is(e[0], target) -} diff --git a/backend/vendor/github.com/hashicorp/go-multierror/prefix.go b/backend/vendor/github.com/hashicorp/go-multierror/prefix.go deleted file mode 100644 index 5c477abe..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/prefix.go +++ /dev/null @@ -1,37 +0,0 @@ -package multierror - -import ( - "fmt" - - "github.com/hashicorp/errwrap" -) - -// Prefix is a helper function that will prefix some text -// to the given error. If the error is a multierror.Error, then -// it will be prefixed to each wrapped error. -// -// This is useful to use when appending multiple multierrors -// together in order to give better scoping. -func Prefix(err error, prefix string) error { - if err == nil { - return nil - } - - format := fmt.Sprintf("%s {{err}}", prefix) - switch err := err.(type) { - case *Error: - // Typed nils can reach here, so initialize if we are nil - if err == nil { - err = new(Error) - } - - // Wrap each of the errors - for i, e := range err.Errors { - err.Errors[i] = errwrap.Wrapf(format, e) - } - - return err - default: - return errwrap.Wrapf(format, err) - } -} diff --git a/backend/vendor/github.com/hashicorp/go-multierror/sort.go b/backend/vendor/github.com/hashicorp/go-multierror/sort.go deleted file mode 100644 index fecb14e8..00000000 --- a/backend/vendor/github.com/hashicorp/go-multierror/sort.go +++ /dev/null @@ -1,16 +0,0 @@ -package multierror - -// Len implements sort.Interface function for length -func (err Error) Len() int { - return len(err.Errors) -} - -// Swap implements sort.Interface function for swapping elements -func (err Error) Swap(i, j int) { - err.Errors[i], err.Errors[j] = err.Errors[j], err.Errors[i] -} - -// Less implements sort.Interface function for determining order -func (err Error) Less(i, j int) bool { - return err.Errors[i].Error() < err.Errors[j].Error() -} diff --git a/backend/vendor/github.com/kyokomi/emoji/v2/emoji_codemap.go b/backend/vendor/github.com/kyokomi/emoji/v2/emoji_codemap.go index a6e039e4..017a5c47 100644 --- a/backend/vendor/github.com/kyokomi/emoji/v2/emoji_codemap.go +++ b/backend/vendor/github.com/kyokomi/emoji/v2/emoji_codemap.go @@ -62,6 +62,7 @@ func emojiCode() map[string]string { ":Leo:": "\u264c", ":Libra:": "\u264e", ":Mrs._Claus:": "\U0001f936", + ":Mx_Claus:": "\U0001f9d1\u200d\U0001f384", ":NEW_button:": "\U0001f195", ":NG_button:": "\U0001f196", ":OK_button:": "\U0001f197", @@ -220,6 +221,7 @@ func emojiCode() map[string]string { ":bald_man:": "\U0001f468\u200d\U0001f9b2", ":bald_person:": "\U0001f9d1\u200d\U0001f9b2", ":bald_woman:": "\U0001f469\u200d\U0001f9b2", + ":ballet_dancer:": "\U0001f9d1\u200d\U0001fa70", ":ballet_shoes:": "\U0001fa70", ":balloon:": "\U0001f388", ":ballot_box:": "\U0001f5f3", @@ -764,6 +766,7 @@ func emojiCode() map[string]string { ":disappointed_face:": "\U0001f61e", ":disappointed_relieved:": "\U0001f625", ":disguised_face:": "\U0001f978", + ":distorted_face:": "\U0001faea", ":divide:": "\u2797", ":dividers:": "\U0001f5c2", ":diving_mask:": "\U0001f93f", @@ -890,6 +893,7 @@ func emojiCode() map[string]string { ":face_savoring_food:": "\U0001f60b", ":face_screaming_in_fear:": "\U0001f631", ":face_vomiting:": "\U0001f92e", + ":face_with_bags_under_eyes:": "\U0001fae9", ":face_with_cowboy_hat:": "\U0001f920", ":face_with_crossed-out_eyes:": "\U0001f635", ":face_with_diagonal_mouth:": "\U0001fae4", @@ -1018,12 +1022,14 @@ func emojiCode() map[string]string { ":ferry:": "\u26f4\ufe0f", ":field_hockey:": "\U0001f3d1", ":field_hockey_stick_and_ball:": "\U0001f3d1", + ":fight_cloud:": "\U0001faef", ":fiji:": "\U0001f1eb\U0001f1ef", ":file_cabinet:": "\U0001f5c4\ufe0f", ":file_folder:": "\U0001f4c1", ":film_frames:": "\U0001f39e\ufe0f", ":film_projector:": "\U0001f4fd\ufe0f", ":film_strip:": "\U0001f39e\ufe0f", + ":fingerprint:": "\U0001fac6", ":fingers_crossed:": "\U0001f91e", ":fingers_crossed_tone1:": "\U0001f91e\U0001f3fb", ":fingers_crossed_tone2:": "\U0001f91e\U0001f3fc", @@ -1251,6 +1257,7 @@ func emojiCode() map[string]string { ":flag-rs:": "\U0001f1f7\U0001f1f8", ":flag-rw:": "\U0001f1f7\U0001f1fc", ":flag-sa:": "\U0001f1f8\U0001f1e6", + ":flag-sark:": "\U0001f1e8\U0001f1f6", ":flag-sb:": "\U0001f1f8\U0001f1e7", ":flag-sc:": "\U0001f1f8\U0001f1e8", ":flag-scotland:": "\U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f", @@ -1504,6 +1511,7 @@ func emojiCode() map[string]string { ":flag_Réunion:": "\U0001f1f7\U0001f1ea", ":flag_Samoa:": "\U0001f1fc\U0001f1f8", ":flag_San_Marino:": "\U0001f1f8\U0001f1f2", + ":flag_Sark:": "\U0001f1e8\U0001f1f6", ":flag_Saudi_Arabia:": "\U0001f1f8\U0001f1e6", ":flag_Scotland:": "\U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f", ":flag_Senegal:": "\U0001f1f8\U0001f1f3", @@ -1995,6 +2003,7 @@ func emojiCode() map[string]string { ":haircut:": "\U0001f487\u200d\u2640\ufe0f", ":haircut_man:": "\U0001f487\u200d\u2642\ufe0f", ":haircut_woman:": "\U0001f487\u200d\u2640\ufe0f", + ":hairy_creature:": "\U0001fac8", ":haiti:": "\U0001f1ed\U0001f1f9", ":hamburger:": "\U0001f354", ":hammer:": "\U0001f528", @@ -2012,43 +2021,44 @@ func emojiCode() map[string]string { ":hand_splayed_tone5:": "\U0001f590\U0001f3ff", ":hand_with_fingers_splayed:": "\U0001f590", ":hand_with_index_finger_and_thumb_crossed:": "\U0001faf0", - ":handbag:": "\U0001f45c", - ":handball:": "\U0001f93e", - ":handball_person:": "\U0001f93e", - ":handshake:": "\U0001f91d", - ":hankey:": "\U0001f4a9", - ":hash:": "#\ufe0f\u20e3", - ":hatched_chick:": "\U0001f425", - ":hatching_chick:": "\U0001f423", - ":head_bandage:": "\U0001f915", - ":head_shaking_horizontally:": "\U0001f642\u200d\u2194\ufe0f", - ":head_shaking_vertically:": "\U0001f642\u200d\u2195\ufe0f", - ":headphone:": "\U0001f3a7", - ":headphones:": "\U0001f3a7", - ":headstone:": "\U0001faa6", - ":health_worker:": "\U0001f9d1\u200d\u2695\ufe0f", - ":hear-no-evil_monkey:": "\U0001f649", - ":hear_no_evil:": "\U0001f649", - ":heard_mcdonald_islands:": "\U0001f1ed\U0001f1f2", - ":heart:": "\u2764\ufe0f", - ":heart_decoration:": "\U0001f49f", - ":heart_exclamation:": "\u2763", - ":heart_eyes:": "\U0001f60d", - ":heart_eyes_cat:": "\U0001f63b", - ":heart_hands:": "\U0001faf6", - ":heart_on_fire:": "\u2764\ufe0f\u200d\U0001f525", - ":heart_suit:": "\u2665", - ":heart_with_arrow:": "\U0001f498", - ":heart_with_ribbon:": "\U0001f49d", - ":heartbeat:": "\U0001f493", - ":heartpulse:": "\U0001f497", - ":hearts:": "\u2665\ufe0f", - ":heavy_check_mark:": "\u2714\ufe0f", - ":heavy_division_sign:": "\u2797", - ":heavy_dollar_sign:": "\U0001f4b2", - ":heavy_equals_sign:": "\U0001f7f0", - ":heavy_exclamation_mark:": "\u2757", - ":heavy_heart_exclamation:": "\u2763\ufe0f", + ":handbag:": "\U0001f45c", + ":handball:": "\U0001f93e", + ":handball_person:": "\U0001f93e", + ":handshake:": "\U0001f91d", + ":hankey:": "\U0001f4a9", + ":harp:": "\U0001fa89", + ":hash:": "#\ufe0f\u20e3", + ":hatched_chick:": "\U0001f425", + ":hatching_chick:": "\U0001f423", + ":head_bandage:": "\U0001f915", + ":head_shaking_horizontally:": "\U0001f642\u200d\u2194\ufe0f", + ":head_shaking_vertically:": "\U0001f642\u200d\u2195\ufe0f", + ":headphone:": "\U0001f3a7", + ":headphones:": "\U0001f3a7", + ":headstone:": "\U0001faa6", + ":health_worker:": "\U0001f9d1\u200d\u2695\ufe0f", + ":hear-no-evil_monkey:": "\U0001f649", + ":hear_no_evil:": "\U0001f649", + ":heard_mcdonald_islands:": "\U0001f1ed\U0001f1f2", + ":heart:": "\u2764\ufe0f", + ":heart_decoration:": "\U0001f49f", + ":heart_exclamation:": "\u2763", + ":heart_eyes:": "\U0001f60d", + ":heart_eyes_cat:": "\U0001f63b", + ":heart_hands:": "\U0001faf6", + ":heart_on_fire:": "\u2764\ufe0f\u200d\U0001f525", + ":heart_suit:": "\u2665", + ":heart_with_arrow:": "\U0001f498", + ":heart_with_ribbon:": "\U0001f49d", + ":heartbeat:": "\U0001f493", + ":heartpulse:": "\U0001f497", + ":hearts:": "\u2665\ufe0f", + ":heavy_check_mark:": "\u2714\ufe0f", + ":heavy_division_sign:": "\u2797", + ":heavy_dollar_sign:": "\U0001f4b2", + ":heavy_equals_sign:": "\U0001f7f0", + ":heavy_exclamation_mark:": "\u2757", + ":heavy_heart_exclamation:": "\u2763\ufe0f", ":heavy_heart_exclamation_mark_ornament:": "\u2763\ufe0f", ":heavy_minus_sign:": "\u2796", ":heavy_multiplication_x:": "\u2716\ufe0f", @@ -2237,6 +2247,7 @@ func emojiCode() map[string]string { ":ladder:": "\U0001fa9c", ":lady_beetle:": "\U0001f41e", ":ladybug:": "\U0001f41e", + ":landslide:": "\U0001f6d8", ":lantern:": "\U0001f3ee", ":laos:": "\U0001f1f1\U0001f1e6", ":laptop:": "\U0001f4bb", @@ -2263,6 +2274,7 @@ func emojiCode() map[string]string { ":latvia:": "\U0001f1f1\U0001f1fb", ":laughing:": "\U0001f606", ":leaf_fluttering_in_wind:": "\U0001f343", + ":leafless_tree:": "\U0001fabe", ":leafy_green:": "\U0001f96c", ":leaves:": "\U0001f343", ":lebanon:": "\U0001f1f1\U0001f1e7", @@ -3146,6 +3158,7 @@ func emojiCode() map[string]string { ":orange_heart:": "\U0001f9e1", ":orange_square:": "\U0001f7e7", ":orangutan:": "\U0001f9a7", + ":orca:": "\U0001facd", ":orthodox_cross:": "\u2626\ufe0f", ":otter:": "\U0001f9a6", ":outbox_tray:": "\U0001f4e4", @@ -3702,6 +3715,7 @@ func emojiCode() map[string]string { ":rolling_on_the_floor_laughing:": "\U0001f923", ":romania:": "\U0001f1f7\U0001f1f4", ":rooster:": "\U0001f413", + ":root_vegetable:": "\U0001fadc", ":rose:": "\U0001f339", ":rosette:": "\U0001f3f5\ufe0f", ":rotating_light:": "\U0001f6a8", @@ -3810,6 +3824,7 @@ func emojiCode() map[string]string { ":shopping_trolley:": "\U0001f6d2", ":shortcake:": "\U0001f370", ":shorts:": "\U0001fa73", + ":shovel:": "\U0001fa8f", ":shower:": "\U0001f6bf", ":shrimp:": "\U0001f990", ":shrug:": "\U0001f937", @@ -3938,6 +3953,7 @@ func emojiCode() map[string]string { ":spiral_note_pad:": "\U0001f5d2\ufe0f", ":spiral_notepad:": "\U0001f5d2", ":spiral_shell:": "\U0001f41a", + ":splatter:": "\U0001fadf", ":spock-hand:": "\U0001f596", ":sponge:": "\U0001f9fd", ":spoon:": "\U0001f944", @@ -4129,6 +4145,7 @@ func emojiCode() map[string]string { ":tram_car:": "\U0001f68b", ":transgender_flag:": "\U0001f3f3\ufe0f\u200d\u26a7\ufe0f", ":transgender_symbol:": "\u26a7\ufe0f", + ":treasure_chest:": "\U0001fa8e", ":triangular_flag:": "\U0001f6a9", ":triangular_flag_on_post:": "\U0001f6a9", ":triangular_ruler:": "\U0001f4d0", @@ -4139,6 +4156,7 @@ func emojiCode() map[string]string { ":triumph:": "\U0001f624", ":troll:": "\U0001f9cc", ":trolleybus:": "\U0001f68e", + ":trombone:": "\U0001fa8a", ":trophy:": "\U0001f3c6", ":tropical_drink:": "\U0001f379", ":tropical_fish:": "\U0001f420", @@ -4881,6 +4899,7 @@ func emojiRevCode() map[string][]string { "\U0001f1e8\U0001f1f3": {":cn:", ":flag_cn:", ":flag_China:"}, "\U0001f1e8\U0001f1f4": {":flag-co:", ":flag_co:", ":colombia:", ":flag_Colombia:"}, "\U0001f1e8\U0001f1f5": {":flag-cp:", ":flag_cp:", ":clipperton_island:", ":flag_Clipperton_Island:"}, + "\U0001f1e8\U0001f1f6": {":flag-sark:", ":flag_Sark:"}, "\U0001f1e8\U0001f1f7": {":flag-cr:", ":flag_cr:", ":costa_rica:", ":flag_Costa_Rica:"}, "\U0001f1e8\U0001f1fa": {":cuba:", ":flag-cu:", ":flag_cu:", ":flag_Cuba:"}, "\U0001f1e8\U0001f1fb": {":flag-cv:", ":flag_cv:", ":cape_verde:", ":flag_Cape_Verde:"}, @@ -5246,7 +5265,7 @@ func emojiRevCode() map[string][]string { "\U0001f381": {":gift:", ":wrapped_gift:"}, "\U0001f382": {":birthday:", ":birthday_cake:"}, "\U0001f383": {":jack-o-lantern:", ":jack_o_lantern:"}, - "\U0001f384": {":Christmas_tree:", ":christmas_tree:"}, + "\U0001f384": {":christmas_tree:", ":Christmas_tree:"}, "\U0001f385": {":santa:", ":Santa_Claus:"}, "\U0001f385\U0001f3fb": {":santa_tone1:"}, "\U0001f385\U0001f3fc": {":santa_tone2:"}, @@ -5465,7 +5484,7 @@ func emojiRevCode() map[string][]string { "\U0001f3ec": {":department_store:"}, "\U0001f3ed": {":factory:"}, "\U0001f3ee": {":lantern:", ":izakaya_lantern:", ":red_paper_lantern:"}, - "\U0001f3ef": {":Japanese_castle:", ":japanese_castle:"}, + "\U0001f3ef": {":japanese_castle:", ":Japanese_castle:"}, "\U0001f3f0": {":castle:", ":european_castle:"}, "\U0001f3f3": {":flag_white:", ":white_flag:"}, "\U0001f3f3\ufe0f": {":waving_white_flag:"}, @@ -5608,7 +5627,7 @@ func emojiRevCode() map[string][]string { "\U0001f44b\U0001f3fd": {":wave_tone3:"}, "\U0001f44b\U0001f3fe": {":wave_tone4:"}, "\U0001f44b\U0001f3ff": {":wave_tone5:"}, - "\U0001f44c": {":OK_hand:", ":ok_hand:"}, + "\U0001f44c": {":ok_hand:", ":OK_hand:"}, "\U0001f44c\U0001f3fb": {":ok_hand_tone1:"}, "\U0001f44c\U0001f3fc": {":ok_hand_tone2:"}, "\U0001f44c\U0001f3fd": {":ok_hand_tone3:"}, @@ -6169,7 +6188,7 @@ func emojiRevCode() map[string][]string { "\U0001f4a1": {":bulb:", ":light_bulb:"}, "\U0001f4a2": {":anger:", ":anger_symbol:"}, "\U0001f4a3": {":bomb:"}, - "\U0001f4a4": {":ZZZ:", ":zzz:"}, + "\U0001f4a4": {":zzz:", ":ZZZ:"}, "\U0001f4a5": {":boom:", ":collision:"}, "\U0001f4a6": {":sweat_drops:", ":sweat_droplets:"}, "\U0001f4a7": {":droplet:"}, @@ -6466,8 +6485,8 @@ func emojiRevCode() map[string][]string { "\U0001f5fa": {":map:"}, "\U0001f5fa\ufe0f": {":world_map:"}, "\U0001f5fb": {":mount_fuji:"}, - "\U0001f5fc": {":Tokyo_tower:", ":tokyo_tower:"}, - "\U0001f5fd": {":Statue_of_Liberty:", ":statue_of_liberty:"}, + "\U0001f5fc": {":tokyo_tower:", ":Tokyo_tower:"}, + "\U0001f5fd": {":statue_of_liberty:", ":Statue_of_Liberty:"}, "\U0001f5fe": {":japan:", ":map_of_Japan:"}, "\U0001f5ff": {":moai:", ":moyai:"}, "\U0001f600": {":grinning:", ":grinning_face:"}, @@ -6544,7 +6563,7 @@ func emojiRevCode() map[string][]string { "\U0001f642\u200d\u2195\ufe0f": {":head_shaking_vertically:"}, "\U0001f643": {":upside_down:", ":upside-down_face:", ":upside_down_face:"}, "\U0001f644": {":roll_eyes:", ":rolling_eyes:", ":face_with_rolling_eyes:"}, - "\U0001f645": {":person_gesturing_NO:", ":person_gesturing_no:"}, + "\U0001f645": {":person_gesturing_no:", ":person_gesturing_NO:"}, "\U0001f645\U0001f3fb": {":person_gesturing_no_tone1:"}, "\U0001f645\U0001f3fb\u200d\u2640\ufe0f": {":woman_gesturing_no_tone1:"}, "\U0001f645\U0001f3fb\u200d\u2642\ufe0f": {":man_gesturing_no_tone1:"}, @@ -6560,9 +6579,9 @@ func emojiRevCode() map[string][]string { "\U0001f645\U0001f3ff": {":person_gesturing_no_tone5:"}, "\U0001f645\U0001f3ff\u200d\u2640\ufe0f": {":woman_gesturing_no_tone5:"}, "\U0001f645\U0001f3ff\u200d\u2642\ufe0f": {":man_gesturing_no_tone5:"}, - "\U0001f645\u200d\u2640\ufe0f": {":no_good:", ":ng_woman:", ":no_good_woman:", ":woman-gesturing-no:", ":woman_gesturing_NO:", ":woman_gesturing_no:"}, - "\U0001f645\u200d\u2642\ufe0f": {":ng_man:", ":no_good_man:", ":man-gesturing-no:", ":man_gesturing_NO:", ":man_gesturing_no:"}, - "\U0001f646": {":ok_person:", ":person_gesturing_OK:", ":person_gesturing_ok:"}, + "\U0001f645\u200d\u2640\ufe0f": {":no_good:", ":ng_woman:", ":no_good_woman:", ":woman-gesturing-no:", ":woman_gesturing_no:", ":woman_gesturing_NO:"}, + "\U0001f645\u200d\u2642\ufe0f": {":ng_man:", ":no_good_man:", ":man-gesturing-no:", ":man_gesturing_no:", ":man_gesturing_NO:"}, + "\U0001f646": {":ok_person:", ":person_gesturing_ok:", ":person_gesturing_OK:"}, "\U0001f646\U0001f3fb": {":person_gesturing_ok_tone1:"}, "\U0001f646\U0001f3fb\u200d\u2640\ufe0f": {":woman_gesturing_ok_tone1:"}, "\U0001f646\U0001f3fb\u200d\u2642\ufe0f": {":man_gesturing_ok_tone1:"}, @@ -6578,8 +6597,8 @@ func emojiRevCode() map[string][]string { "\U0001f646\U0001f3ff": {":person_gesturing_ok_tone5:"}, "\U0001f646\U0001f3ff\u200d\u2640\ufe0f": {":woman_gesturing_ok_tone5:"}, "\U0001f646\U0001f3ff\u200d\u2642\ufe0f": {":man_gesturing_ok_tone5:"}, - "\U0001f646\u200d\u2640\ufe0f": {":ok_woman:", ":woman-gesturing-ok:", ":woman_gesturing_OK:", ":woman_gesturing_ok:"}, - "\U0001f646\u200d\u2642\ufe0f": {":ok_man:", ":man-gesturing-ok:", ":man_gesturing_OK:", ":man_gesturing_ok:"}, + "\U0001f646\u200d\u2640\ufe0f": {":ok_woman:", ":woman-gesturing-ok:", ":woman_gesturing_ok:", ":woman_gesturing_OK:"}, + "\U0001f646\u200d\u2642\ufe0f": {":ok_man:", ":man-gesturing-ok:", ":man_gesturing_ok:", ":man_gesturing_OK:"}, "\U0001f647": {":bow:", ":person_bowing:"}, "\U0001f647\U0001f3fb": {":person_bowing_tone1:"}, "\U0001f647\U0001f3fb\u200d\u2640\ufe0f": {":woman_bowing_tone1:"}, @@ -6831,6 +6850,7 @@ func emojiRevCode() map[string][]string { "\U0001f6d5": {":hindu_temple:"}, "\U0001f6d6": {":hut:"}, "\U0001f6d7": {":elevator:"}, + "\U0001f6d8": {":landslide:"}, "\U0001f6dc": {":wireless:"}, "\U0001f6dd": {":playground_slide:"}, "\U0001f6de": {":wheel:"}, @@ -7186,7 +7206,7 @@ func emojiRevCode() map[string][]string { "\U0001f993": {":zebra:", ":zebra_face:"}, "\U0001f994": {":hedgehog:"}, "\U0001f995": {":sauropod:"}, - "\U0001f996": {":T-Rex:", ":t-rex:", ":t_rex:"}, + "\U0001f996": {":t-rex:", ":T-Rex:", ":t_rex:"}, "\U0001f997": {":cricket:"}, "\U0001f998": {":kangaroo:"}, "\U0001f999": {":llama:"}, @@ -7267,7 +7287,7 @@ func emojiRevCode() map[string][]string { "\U0001f9d1\u200d\U0001f33e": {":farmer:"}, "\U0001f9d1\u200d\U0001f373": {":cook:"}, "\U0001f9d1\u200d\U0001f37c": {":person_feeding_baby:"}, - "\U0001f9d1\u200d\U0001f384": {":mx_claus:"}, + "\U0001f9d1\u200d\U0001f384": {":mx_claus:", ":Mx_Claus:"}, "\U0001f9d1\u200d\U0001f393": {":student:"}, "\U0001f9d1\u200d\U0001f3a4": {":singer:"}, "\U0001f9d1\u200d\U0001f3a8": {":artist:"}, @@ -7294,6 +7314,7 @@ func emojiRevCode() map[string][]string { "\U0001f9d1\u200d\U0001f9d1\u200d\U0001f9d2\u200d\U0001f9d2": {":family_adult_adult_child_child:"}, "\U0001f9d1\u200d\U0001f9d2": {":family_adult_child:"}, "\U0001f9d1\u200d\U0001f9d2\u200d\U0001f9d2": {":family_adult_child_child:"}, + "\U0001f9d1\u200d\U0001fa70": {":ballet_dancer:"}, "\U0001f9d1\u200d\u2695\ufe0f": {":health_worker:"}, "\U0001f9d1\u200d\u2696\ufe0f": {":judge:"}, "\U0001f9d1\u200d\u2708\ufe0f": {":pilot:"}, @@ -7520,6 +7541,10 @@ func emojiRevCode() map[string][]string { "\U0001fa86": {":nesting_dolls:"}, "\U0001fa87": {":maracas:"}, "\U0001fa88": {":flute:"}, + "\U0001fa89": {":harp:"}, + "\U0001fa8a": {":trombone:"}, + "\U0001fa8e": {":treasure_chest:"}, + "\U0001fa8f": {":shovel:"}, "\U0001fa90": {":ringed_planet:"}, "\U0001fa91": {":chair:"}, "\U0001fa92": {":razor:"}, @@ -7566,6 +7591,7 @@ func emojiRevCode() map[string][]string { "\U0001fabb": {":hyacinth:"}, "\U0001fabc": {":jellyfish:"}, "\U0001fabd": {":wing:"}, + "\U0001fabe": {":leafless_tree:"}, "\U0001fabf": {":goose:"}, "\U0001fac0": {":anatomical_heart:"}, "\U0001fac1": {":lungs:"}, @@ -7573,6 +7599,9 @@ func emojiRevCode() map[string][]string { "\U0001fac3": {":pregnant_man:"}, "\U0001fac4": {":pregnant_person:"}, "\U0001fac5": {":person_with_crown:"}, + "\U0001fac6": {":fingerprint:"}, + "\U0001fac8": {":hairy_creature:"}, + "\U0001facd": {":orca:"}, "\U0001face": {":moose:"}, "\U0001facf": {":donkey:"}, "\U0001fad0": {":blueberries:"}, @@ -7587,6 +7616,8 @@ func emojiRevCode() map[string][]string { "\U0001fad9": {":jar:"}, "\U0001fada": {":ginger_root:"}, "\U0001fadb": {":pea_pod:"}, + "\U0001fadc": {":root_vegetable:"}, + "\U0001fadf": {":splatter:"}, "\U0001fae0": {":melting_face:"}, "\U0001fae1": {":saluting_face:"}, "\U0001fae2": {":face_with_open_eyes_and_hand_over_mouth:"}, @@ -7596,6 +7627,9 @@ func emojiRevCode() map[string][]string { "\U0001fae6": {":biting_lip:"}, "\U0001fae7": {":bubbles:"}, "\U0001fae8": {":shaking_face:"}, + "\U0001fae9": {":face_with_bags_under_eyes:"}, + "\U0001faea": {":distorted_face:"}, + "\U0001faef": {":fight_cloud:"}, "\U0001faf0": {":hand_with_index_finger_and_thumb_crossed:"}, "\U0001faf1": {":rightwards_hand:"}, "\U0001faf2": {":leftwards_hand:"}, @@ -7709,18 +7743,18 @@ func emojiRevCode() map[string][]string { "\u263a\ufe0f": {":relaxed:"}, "\u2640\ufe0f": {":female_sign:"}, "\u2642\ufe0f": {":male_sign:"}, - "\u2648": {":Aries:", ":aries:"}, - "\u2649": {":Taurus:", ":taurus:"}, - "\u264a": {":Gemini:", ":gemini:"}, - "\u264b": {":Cancer:", ":cancer:"}, - "\u264c": {":Leo:", ":leo:"}, - "\u264d": {":Virgo:", ":virgo:"}, - "\u264e": {":Libra:", ":libra:"}, + "\u2648": {":aries:", ":Aries:"}, + "\u2649": {":taurus:", ":Taurus:"}, + "\u264a": {":gemini:", ":Gemini:"}, + "\u264b": {":cancer:", ":Cancer:"}, + "\u264c": {":leo:", ":Leo:"}, + "\u264d": {":virgo:", ":Virgo:"}, + "\u264e": {":libra:", ":Libra:"}, "\u264f": {":Scorpio:", ":scorpius:"}, - "\u2650": {":Sagittarius:", ":sagittarius:"}, - "\u2651": {":Capricorn:", ":capricorn:"}, - "\u2652": {":Aquarius:", ":aquarius:"}, - "\u2653": {":Pisces:", ":pisces:"}, + "\u2650": {":sagittarius:", ":Sagittarius:"}, + "\u2651": {":capricorn:", ":Capricorn:"}, + "\u2652": {":aquarius:", ":Aquarius:"}, + "\u2653": {":pisces:", ":Pisces:"}, "\u265f\ufe0f": {":chess_pawn:"}, "\u2660": {":spade_suit:"}, "\u2660\ufe0f": {":spades:"}, @@ -7763,7 +7797,7 @@ func emojiRevCode() map[string][]string { "\u26c5": {":partly_sunny:", ":sun_behind_cloud:"}, "\u26c8": {":thunder_cloud_rain:", ":cloud_with_lightning_and_rain:"}, "\u26c8\ufe0f": {":thunder_cloud_and_rain:"}, - "\u26ce": {":Ophiuchus:", ":ophiuchus:"}, + "\u26ce": {":ophiuchus:", ":Ophiuchus:"}, "\u26cf\ufe0f": {":pick:"}, "\u26d1": {":helmet_with_cross:", ":rescue_worker’s_helmet:"}, "\u26d1\ufe0f": {":rescue_worker_helmet:", ":helmet_with_white_cross:"}, diff --git a/backend/vendor/github.com/redis/go-redis/v9/.gitignore b/backend/vendor/github.com/redis/go-redis/v9/.gitignore index 93affec7..4134a1f4 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/.gitignore +++ b/backend/vendor/github.com/redis/go-redis/v9/.gitignore @@ -17,3 +17,12 @@ maintenanceNotifications/docs/ # Docker-generated files (TLS certificates, cluster data, etc.) dockers/*/tls/ dockers/osscluster-tls/ + +# Per-user Claude Code settings (machine/personal overrides, not shared policy) +.claude/settings.local.json + +# example build artifacts +example/autopipeline/autopipeline + +# Claude Code worktrees (ephemeral, local) +.claude/worktrees/ diff --git a/backend/vendor/github.com/redis/go-redis/v9/AGENTS.md b/backend/vendor/github.com/redis/go-redis/v9/AGENTS.md new file mode 100644 index 00000000..ca0a3116 --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/AGENTS.md @@ -0,0 +1,228 @@ +# AGENTS.md + +Guidance for AI coding agents (and humans) working in this repository. This is +the shared, tool-agnostic source of truth: Claude Code loads it through +`CLAUDE.md` (which imports this file), and other agents (Codex, Cursor, Aider, +Zed, …) read `AGENTS.md` directly. Edit repository guidance here, not in +`CLAUDE.md`. + +## Repository + +go-redis is the official Redis client for Go. Module path: +`github.com/redis/go-redis/v9` (Go 1.24+). The repo is a multi-module workspace +— every directory containing a `go.mod` is built and tested independently: + +- root (`github.com/redis/go-redis/v9`) — the client library. +- `extra/redisotel`, `extra/redisotel-native`, `extra/redisprometheus`, + `extra/rediscensus`, `extra/rediscmd` — instrumentation adapters with their + own module paths (so they can pin large telemetry deps without forcing them on + root consumers). +- `internal/customvet` — custom `go vet` analyzers (also its own module). +- `maintnotifications/e2e`, `doctests`, `fuzz`, examples under `example/` — + separate modules. + +The Makefile iterates over every `go.mod` (`GO_MOD_DIRS`) when running +`test.ci`, `go_mod_tidy`, etc. When you add a dependency in one module, you +almost never need to update the others. + +## Common commands + +Tests run against a Redis stack started via Docker Compose. Profiles in +`docker-compose.yml` control which services come up (`standalone`, `cluster`, +`sentinel`, `all`, `e2e`). + +```sh +make docker.start # bring up the full test stack (profile: all) +make docker.stop +make test # docker.start -> test.ci -> docker.stop +make test.ci # run tests assuming containers are already up +make test.ci.skip-vectorsets # when REDIS_VERSION < 8 +make bench # go test -bench=. (root module only) +make fmt # gofumpt + goimports -local github.com/redis/go-redis +make build +make go_mod_tidy # go mod tidy across every module +``` + +E2E (maintenance notifications) needs the extra `cae-resp-proxy` service: + +```sh +make test.e2e # starts e2e profile, runs ./maintnotifications/e2e/, tears down +make test.e2e.docker # subset that runs inside docker +make test.e2e.logic # logic-only tests, no proxy required +``` + +Run a single test. The root suite is Ginkgo-based (`bsm/ginkgo` + `bsm/gomega` +forks), so `go test -run` matches the Go-level wrapper and you focus a spec with +the Ginkgo flag: + +```sh +go test -run TestGinkgoSuite . -ginkgo.focus="ZAdd" +go test -run TestGinkgoSuite . -ginkgo.focus="cluster" +``` + +Plain `go test` tests (most files outside the Ginkgo suite, e.g. `internal/...`, +`maintnotifications/...`) work the usual way: + +```sh +go test -run TestConnStateMachine ./internal/pool/... +go test -race -run TestCircuitBreaker ./maintnotifications/... +``` + +Env knobs (passed through the Makefile): + +- `REDIS_VERSION` — e.g. `8.8`. Drives both the test image tag and + `main_test.go` version-gating (`SkipBeforeRedisVersion` / + `SkipAfterRedisVersion`). +- `CLIENT_LIBS_TEST_IMAGE` — full image ref, e.g. + `redislabs/client-libs-test:8.8-m03`. +- `RE_CLUSTER=true` — run against a Redis Enterprise cluster instead of the + docker-compose stack (the suite then skips ring/sentinel/TLS-cluster setup). +- `RCE_DOCKER=true` — Redis CE in docker (default for `make test`). +- `REDIS_PORT` — override the default standalone port (`6380`). + +CI also runs the custom vet tool: +`go vet -vettool ./internal/customvet/customvet ./...`. The `setval` analyzer +requires every `Cmder` with a `Result()` to also have a `SetVal()`. + +## Architecture + +### Client types (root package) + +All clients are in the root package and share most plumbing: + +- `Client` (`redis.go`) — single-node client. +- `ClusterClient` (`osscluster.go`) — Redis Cluster aware. `osscluster_router.go` + routes commands to the right shard; `internal/routing/` handles cluster-wide + aggregation policies (e.g. fan-out for `KEYS`, `DBSIZE`). +- `Ring` (`ring.go`) — client-side sharding across independent Redis nodes + (consistent hashing, no cluster protocol). +- Failover client (`sentinel.go`) — Sentinel-managed failover. +- `UniversalClient` (`universal.go`) — wrapper that picks one of the above based + on options. + +Command surface lives in topical files: `string_commands.go`, `hash_commands.go`, +`stream_commands.go`, `search_commands.go`, `vectorset_commands.go`, etc. Each +file defines methods on the shared `Cmdable` interface so every client type gets +the same API. + +### Hooks (`redis.go` `hooksMixin`) + +Three hook chains run around every operation: `DialHook`, `ProcessHook`, +`ProcessPipelineHook`. Hooks are registered via `client.AddHook(...)` and chain +in FIFO order; each hook must call `next` to continue. When a hook wraps an +error, it must call `cmd.SetErr(wrappedErr)` so the typed-error helpers +(`redis.IsLoadingError`, `IsMovedError`, etc. in `error.go`) keep working through +`errors.As`. The README has a longer pipeline-hook example. + +### Connection pool (`internal/pool`) + +Owns dialing, idle/active connection bookkeeping, conn state (`conn_state.go`), +pubsub-conn lifecycle (`pubsub.go`), and the dial-retry/backoff logic that powers +`DialerRetries` / `DialerRetryBackoff` (also exposed at `dial_retry_backoff.go` +in the root). `OnConnect`, `MinIdleConns`, and the buffer-size options +(`ReadBufferSize`/`WriteBufferSize`, default 32 KiB since v9.12) flow through +here. + +### Protocol (`internal/proto`) + +RESP2/RESP3 reader and writer. Push notifications (RESP3 `>`-prefixed frames) are +peeked here and dispatched via the `push/` package. The `push.Registry` lets +callers register handlers for specific notification names; +`maintnotifications/push_notification_handler.go` is how `maintnotifications` +plugs in. + +### Maintenance notifications (`maintnotifications/`) + +This is a non-trivial subsystem worth understanding before touching +cluster/handoff code. It listens for RESP3 push notifications about cluster +maintenance (`MOVING`, `MIGRATING`, `MIGRATED`, `FAILING_OVER`, `FAILED_OVER` +for standalone; `SMIGRATING`, `SMIGRATED` for cluster) and performs seamless +connection handoff to new endpoints. Key pieces: + +- `manager.go` — coordinates state transitions. +- `handoff_worker.go` — moves in-flight ops to new connections. +- `pool_hook.go` — integrates with `internal/pool` to mark/replace connections. +- `circuit_breaker.go` — backs off when the upstream is unhealthy. +- `state.go` — per-connection state machine. +- E2E coverage lives in `maintnotifications/e2e/` and drives a fault-injector / + RESP proxy (`cae-resp-proxy`). + +Configuration is via `redis.Options.MaintNotificationsConfig`; modes are +`ModeAuto` (default), `ModeEnabled` (require server support), `ModeDisabled`. +RESP3 (`Protocol: 3`) is required. + +### Authentication (`auth/`, `internal/auth/streaming`) + +Four credential sources, in priority order: streaming provider (e.g. Entra ID +via `go-redis-entraid`), context-based provider, function provider, static +`Username`/`Password`. The streaming provider is what enables token rotation +without reconnecting — the listener in `auth/reauth_credentials_listener.go` +issues `AUTH` on each refresh. + +### Internal helpers + +- `internal/hscan` — struct scanning for `HGETALL` results (`Scan` interface + re-exported as `redis.Scanner`). +- `internal/hashtag` — extracts `{tag}` segments for cluster slot routing. +- `internal/routing` — aggregator policies and shard pickers used by + `ClusterClient` for multi-shard commands. +- `internal/otel` — small OpenTelemetry shim used to keep root free of telemetry + deps; full instrumentation lives in `extra/redisotel-native`. + +## Architectural specs + +Read the relevant design doc **before** changing code in that subsystem. They +cover invariants and decisions that aren't obvious from the code, and are plain +markdown any tool or editor can open: + +- `.claude/specs/pool.md` — connection pool: `wantConn` queue and FIFO + discipline, `ConnState` machine, dial retry/backoff, hook integration, the + re-auth/handoff coexistence contract. +- `.claude/specs/cluster-routing.md` — slot computation, MOVED/ASK redirection, + request/response policies, aggregators, replica routing, topology reload, + cross-slot rules. +- `.claude/specs/maintnotifications.md` — RESP3 push notification protocol, mode + handshake, per-conn state, handoff worker pool, circuit breaker, endpoint-type + resolution, cluster vs. standalone differences. + +## Conventions + +- New `Cmder` type → also implement `SetVal` (the custom vet `setval` check + enforces this; `SetErr` is on the embedded `baseCmd`). +- Wrap errors with custom error types that implement `Unwrap`, or use + `fmt.Errorf("...: %w", err)`. Always call `cmd.SetErr(...)` after wrapping so + typed-error checks still pass. +- `gofumpt` + `goimports -local github.com/redis/go-redis` is the formatter + (`make fmt`); CI runs both. +- Don't log directly — use `internal.Logger` (set via `redis.SetLogger`); + `logging.Disable()` is called in tests. +- Version-gate Redis-version-specific tests with `SkipBeforeRedisVersion` / + `SkipAfterRedisVersion` rather than skipping at the suite level. + +### Commits and PRs + +Conventional Commits, short and exact — `(): `. +Subject ≤50 chars (hard cap 72), imperative ("add", not "added"), no trailing +period. Body only when the *why* isn't obvious from the diff; wrap at 72. + +- Types: `feat`, `fix`, `refactor`, `perf`, `docs`, `test`, `chore` (also + `build`, `ci`, `style`, `revert`). +- Scope = the subsystem touched, lowercase: `pool`, `conn`, `pubsub`, + `sentinel`, `retry`, `command`/`cmd`, `vectorset`, `otel`, `streams`, `push`, + `deps`, `ci`, `tests`, `docs`. Omit only for genuinely cross-cutting changes. +- Breaking change: `feat(scope)!: ...` plus a `BREAKING CHANGE:` body line. + Reference issues/PRs at the end — `Closes #42`, `Refs #17`. +- **No AI-attribution trailer.** Do not add `Co-Authored-By: …`, "Generated with + …", or any AI-attribution line to commits or PR bodies in this repo. + +## Repo-specific tooling + +`.claude/` holds shared AI config: + +- `commands/` — slash commands (e.g. `check-ci`, which summarizes a PR's CI). +- `skills/` — task playbooks: `testing`, `add-command`, `commit-style`, + `update-ci-image`, `prepare-release`. +- `specs/` — the architecture docs listed above. + +For Claude Code, the skills auto-trigger from their descriptions. For other +tools, each `SKILL.md` is plain markdown you can open and follow directly. diff --git a/backend/vendor/github.com/redis/go-redis/v9/CLAUDE.md b/backend/vendor/github.com/redis/go-redis/v9/CLAUDE.md new file mode 100644 index 00000000..7e9eb08c --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/CLAUDE.md @@ -0,0 +1,19 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +Repository guidance — layout, commands, architecture, and conventions — is +maintained in [`AGENTS.md`](AGENTS.md) so it stays shared across AI tools. It is +imported below; edit `AGENTS.md`, not this file, for that content. + +@AGENTS.md + +## Claude-specific + +- Repo-local **skills** under `.claude/skills/` auto-trigger from their + descriptions — no need to invoke them manually (the set is listed in + `AGENTS.md`). +- **Slash commands** under `.claude/commands/` (e.g. `/check-ci`) are available + in-session. +- Architectural **specs** under `.claude/specs/` are read on demand; open the + relevant one before changing that subsystem. diff --git a/backend/vendor/github.com/redis/go-redis/v9/CONTRIBUTING.md b/backend/vendor/github.com/redis/go-redis/v9/CONTRIBUTING.md index 8c68c522..9f175644 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/CONTRIBUTING.md +++ b/backend/vendor/github.com/redis/go-redis/v9/CONTRIBUTING.md @@ -37,7 +37,7 @@ Here's how to get started with your code contribution: > Note: this clones and builds the docker containers specified in `docker-compose.yml`, to understand more about > the infrastructure that will be started you can check the `docker-compose.yml`. You also have the possiblity > to specify the redis image that will be pulled with the env variable `CLIENT_LIBS_TEST_IMAGE`. -> By default the docker image that will be pulled and started is `redislabs/client-libs-test:8.2.1-pre`. +> By default the docker image that will be pulled and started is `redislabs/client-libs-test:8.10.0`. > If you want to test with newer Redis version, using a newer version of `redislabs/client-libs-test` should work out of the box. 4. While developing, make sure the tests pass by running `make test` (if you have the docker containers running, `make test.ci` may be sufficient). diff --git a/backend/vendor/github.com/redis/go-redis/v9/Makefile b/backend/vendor/github.com/redis/go-redis/v9/Makefile index 1322bd4f..8881f2d3 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/Makefile +++ b/backend/vendor/github.com/redis/go-redis/v9/Makefile @@ -1,8 +1,8 @@ GO_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | sort) -REDIS_VERSION ?= 8.8 +REDIS_VERSION ?= 8.10 RE_CLUSTER ?= false RCE_DOCKER ?= true -CLIENT_LIBS_TEST_IMAGE ?= redislabs/client-libs-test:8.8.0 +CLIENT_LIBS_TEST_IMAGE ?= redislabs/client-libs-test:8.10.0 docker.start: export RE_CLUSTER=$(RE_CLUSTER) && \ @@ -71,6 +71,35 @@ test.ci.skip-vectorsets: cd internal/customvet && go build . go vet -vettool ./internal/customvet/customvet +# Replay the parametrized command integration suites through the AutoPipeliner +# Cmdable faces (blocking + async) to prove the commands behave the same +# batched as on a plain client. Selected via GOREDIS_TEST_SUBJECT (see +# newUniversalSubject in main_test.go). Requires the docker env +# (make docker.start). +# +# The focus covers every parametrized suite (some via the Commands/BitCount +# substrings — Describe names differ in case). Skipped on purpose: DDL Commands (needs the cluster topology the +# lightweight BeforeSuite doesn't build), HotKeys Commands (same), and +# AutoPipeline Blocking Commands (the AP suite itself — running it through an +# AP subject would nest engines). +test.autopipeline-subjects: + # RE_CLUSTER=true forces the lightweight BeforeSuite (no sentinel/ring/cluster + # setup): the command suite only needs the standalone Redis, and running the + # full stateful BeforeSuite twice (once per subject) against the same server + # corrupts replication state and fails the second run. Both faces then run + # cleanly against the same env. The explicit -timeout keeps a hang from + # eating go test's 10m default per subject. + set -e; for subj in ap-blocking ap-async; do \ + echo "=== command suite via GOREDIS_TEST_SUBJECT=$$subj ==="; \ + (export RE_CLUSTER=true && \ + export RCE_DOCKER=$(RCE_DOCKER) && \ + export REDIS_VERSION=$(REDIS_VERSION) && \ + export GOREDIS_TEST_SUBJECT=$$subj && \ + go test -v . -race -skip Example -run TestGinkgoSuite -timeout 8m \ + -ginkgo.focus='Commands|RediSearch commands|Probabilistic commands|RedisTimeseries commands|Redis VectorSet commands|BitCount|ScanIterator|Advanced JSON' \ + -ginkgo.skip='AutoPipeline Blocking Commands|DDL Commands|HotKeys Commands'); \ + done + bench: export RE_CLUSTER=$(RE_CLUSTER) && \ export RCE_DOCKER=$(RCE_DOCKER) && \ @@ -101,7 +130,7 @@ test.e2e.logic: go test -v -run "TestCreateTestFaultInjectorLogic|TestFaultInjectorClientCreation" ./maintnotifications/e2e/ @echo "Logic tests completed!" -.PHONY: all test test.ci test.ci.skip-vectorsets bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop +.PHONY: all test test.ci test.ci.skip-vectorsets test.autopipeline-subjects bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop build: export RE_CLUSTER=$(RE_CLUSTER) && \ diff --git a/backend/vendor/github.com/redis/go-redis/v9/README.md b/backend/vendor/github.com/redis/go-redis/v9/README.md index ae90d2b7..59a2eee8 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/README.md +++ b/backend/vendor/github.com/redis/go-redis/v9/README.md @@ -21,6 +21,7 @@ In `go-redis` we are aiming to support the last three releases of Redis. Current - [Redis 8.2](https://raw.githubusercontent.com/redis/redis/8.2/00-RELEASENOTES) - using Redis CE 8.2 - [Redis 8.4](https://raw.githubusercontent.com/redis/redis/8.4/00-RELEASENOTES) - using Redis CE 8.4 - [Redis 8.8](https://raw.githubusercontent.com/redis/redis/8.8/00-RELEASENOTES) - using Redis CE 8.8 +- [Redis 8.10](https://raw.githubusercontent.com/redis/redis/8.10/00-RELEASENOTES) - using Redis CE 8.10 Although the `go.mod` states it requires at minimum `go 1.24`, our CI is configured to run the tests against all supported versions of Redis and multiple versions of Go ([1.24](https://go.dev/doc/devel/release#go1.24.0), oldstable, and stable). We observe that some modules related test may not pass with @@ -78,9 +79,13 @@ surface. The API is experimental and may change in a future release. - [StreamingCredentialsProvider (e.g. entra id, oauth)](#1-streaming-credentials-provider-highest-priority) (experimental) - [Pub/Sub](https://redis.uptrace.dev/guide/go-redis-pubsub.html). - [Pipelines and transactions](https://redis.uptrace.dev/guide/go-redis-pipelines.html). +- [Automatic pipelining](#automatic-pipelining) (experimental) — batches concurrent + commands into pipelines for you; meant for high-throughput / high-load / scale + use cases. - [Scripting](https://redis.uptrace.dev/guide/lua-scripting.html). - [Redis Sentinel](https://redis.uptrace.dev/guide/go-redis-sentinel.html). - [Redis Cluster](https://redis.uptrace.dev/guide/go-redis-cluster.html). +- [Client-side caching](#client-side-caching). - [Redis Performance Monitoring](https://redis.uptrace.dev/guide/redis-performance-monitoring.html). - [Redis Probabilistic [RedisStack]](https://redis.io/docs/data-types/probabilistic/) - [Customizable read and write buffers size.](#custom-buffer-sizes) @@ -285,6 +290,50 @@ rdb := redis.NewClient(&redis.Options{ }) ``` +### Client-side caching + +go-redis supports server-assisted client-side caching for standalone clients. +Eligible read replies are stored in the application's memory, so repeated reads +can avoid a Redis round trip. Redis tracks which keys each connection has read +and sends RESP3 invalidation notifications when those keys change. go-redis +uses those notifications to evict affected entries automatically. + +> **Experimental:** The client-side caching API may change in a minor release. + +Enable the built-in bounded cache with `ClientSideCacheConfig`: + +```go +rdb := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Protocol: 3, + DB: 0, + ClientSideCacheConfig: &redis.ClientSideCacheConfig{ + MaxEntries: 10_000, + }, +}) +defer rdb.Close() +``` + +Client-side caching currently requires RESP3, a standalone client, and database +0. Fixed `Username` and `Password` values are supported. It is disabled when a +dynamic credential provider is configured, because cached data must never be +reused after the client's ACL identity changes. Only deterministic read +commands supported by the cache are stored; writes and streaming responses +bypass it. + +While client-side caching is enabled, go-redis rejects `SELECT`, `AUTH`, +`HELLO` with arguments, `RESET`, `CLIENT TRACKING`, and raw `SUBSCRIBE`, +`PSUBSCRIBE`, or `SSUBSCRIBE` commands because they would change connection +state that the cache relies on. A guarded command also fails its whole +pipeline. The typed `Subscribe`, `PSubscribe`, and `SSubscribe` APIs remain +supported because they use dedicated connections. + +Invalidations are processed asynchronously. `DrainInterval` controls how often +idle connections are checked for them, while `MaxStaleness` can provide an +optional upper bound on an entry's lifetime. See the +[client-side caching example](./example/client-side-caching) for a working +demonstration. + ### Connecting via a redis url go-redis also supports connecting via the @@ -340,6 +389,118 @@ rdb := redis.NewClient(&redis.Options{ }) ``` +### Automatic pipelining + +**Experimental** — the API may still change. Reach for autopipelining in +high-throughput / high-load / scale scenarios; at low concurrency a plain +client is simpler and just as fast. A runnable usage tour and throughput +comparison live in [`example/autopipeline`](example/autopipeline). + +> **EXPERIMENTAL:** the autopipelining API is subject to change in a future +> release as we gather feedback — pin your go-redis version if you adopt it. + +When many goroutines issue commands concurrently, autopipelining batches them +into Redis pipelines automatically — without you writing any pipeline code. It +comes in two faces: + +- **`AutoPipeline()` — blocking, drop-in.** Each command call blocks until it + executes and returns its own value/error, exactly like a normal client, so + existing code keeps working unchanged. Under concurrency the engine coalesces + commands from all goroutines into deep, back-to-back pipelines (a single + ordered batch stream by default), reaching several times a plain client's + executed commands per second in the same environment — roughly an order of + magnitude with a parallel-batch config (`MaxConcurrentBatches` > 1 with + `Unordered`). Per-goroutine ordering is preserved. +- **`AsyncAutoPipeline()` — deferred, highest throughput.** Command calls return + immediately; you submit a window of commands and read their results afterward, + which keeps each pipeline deep — tens of times a plain client's throughput. + Ordered by default. Absolute numbers depend heavily on the machine, network + path and server; see `autopipeline_bench_README.md` for the benchmark + methodology and multipliers. + +```go +rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379"}) +defer rdb.Close() +ctx := context.Background() + +// Blocking face: drop-in for a normal client, batched under the hood. +ap, err := rdb.AutoPipeline() +if err != nil { // invalid AutoPipelineOptions, or the client is closed + log.Fatal(err) +} +defer ap.Close() + +var wg sync.WaitGroup +for i := 0; i < 1000; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + key := fmt.Sprintf("key:%d", i) + if err := ap.Set(ctx, key, i, 0).Err(); err != nil { // blocks until executed + log.Printf("set %s: %v", key, err) + } + }(i) +} +wg.Wait() +``` + +For maximum throughput, submit a window on the async face and read later: + +```go +ctx := context.Background() +ap, err := rdb.AsyncAutoPipeline() // ordered by default +if err != nil { + log.Fatal(err) +} +defer ap.Close() + +cmds := make([]*redis.StatusCmd, 0, 200) +for i := 0; i < 200; i++ { + cmds = append(cmds, ap.Set(ctx, fmt.Sprintf("key:%d", i), i, 0)) // returns immediately +} +for _, cmd := range cmds { + if err := cmd.Err(); err != nil { // blocks until executed + log.Printf("set: %v", err) + } +} +``` + +Each face has a no-argument form that uses `Options.AutoPipelineOptions` (or the +built-in default) and a `WithOptions` form that takes an explicit +`*AutoPipelineOptions`; both return `(*AutoPipeliner, error)` — the error is +non-nil for an invalid config or a closed client (e.g. +`ap, err := rdb.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{MaxConcurrentBatches: 8, Unordered: true})`); +a handful of parallel batches saturates the link — more permits only add +overlapping batches without deepening them. +They work on `ClusterClient` too: commands are routed to the correct shard per +key, so a single batch may span many slots; ordering across nodes is per key +(same-key commands stay in order, different nodes' sub-pipelines run +concurrently). Because batches share a few pipeline connections, autopipelining +also needs far fewer connections than a plain client at the same concurrency +(see `PipelinePoolSize`). Autopipelining is only a win under concurrency (or +windowed submission) — a single goroutine issuing one blocking command at a +time sees little benefit, and a hand-written `Pipeline()` is still fastest when +you can batch by hand. + +Caveats: a command's context is not honored once it is queued (batches execute +on the autopipeliner's own context) — use a plain client for per-command +deadlines. Blocking commands (`BLPOP`, `WAIT`, ...) are never batched and run +directly on your context — as are `SHUTDOWN` and `MONITOR`, which would +poison a shared pipeline connection — and `Do` also bypasses batching with plain +`Client.Do` semantics — prefer the typed methods (`ap.Set`, `ap.Get`, ...). On +a dropped connection a batch is retried whole (up to `MaxRetries`), so +non-idempotent commands may execute twice. Both faces return a cached, +client-shared instance: the first call's config wins and `Close` stops it for +all callers. Hooks may read command results (the engine hands a hook running +on the dispatch goroutine the same view a plain pipeline hook gets), but a +hook must never issue a command on the same autopipeliner and wait for it — +the nested command needs the very dispatch slot the hook is holding, and the +engine only recovers by failing that flush after its 30s permit backstops. +`Options.Limiter` is consulted once per batch dispatch (as with a manual +pipeline), not once per command. An autopipeliner created on a +`WithTimeout`/`WithReadTimeout` clone is not stopped by the parent's `Close` — +close it explicitly. + ### Advanced Configuration go-redis supports extending the client identification phase to allow projects to send their own custom client identification. @@ -457,6 +618,24 @@ vals, err := rdb.Eval(ctx, "return {KEYS[1],ARGV[1]}", []string{"key"}, "hello") res, err := rdb.Do(ctx, "set", "key", "value").Result() ``` +### Raw commands and connection state + +`Do` sends the command verbatim on whichever pooled connection happens to be +free. For keyspace commands that is all you need. It is the wrong tool for +any command that alters **connection session state** — `SELECT`, +`CLIENT SETNAME`, `CLIENT TRACKING`, `RESET`, `HIMPORT PREPARE`/`DISCARD`, +and similar: the state lands on (or is wiped from) a single arbitrary +connection, later commands are served by other connections that don't share +it, and the affected connection eventually returns to the pool and serves +unrelated callers. The result is nondeterministic behavior that typed APIs +manage for you — for example, the typed `HImport*` methods keep a +client-side registry and replay fieldsets onto every connection that needs +them, while a raw `Do(ctx, "himport", "prepare", ...)` bypasses that +entirely, with no replay, recovery, or discard propagation. + +For session-scoped work without a typed API, hold a dedicated connection +(`client.Conn()`) for its whole lifetime and close it afterwards. + ## Typed Errors go-redis provides typed error checking functions for common Redis errors: diff --git a/backend/vendor/github.com/redis/go-redis/v9/RELEASE-NOTES.md b/backend/vendor/github.com/redis/go-redis/v9/RELEASE-NOTES.md index a125344d..0ebed092 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/RELEASE-NOTES.md +++ b/backend/vendor/github.com/redis/go-redis/v9/RELEASE-NOTES.md @@ -1,5 +1,244 @@ # Release Notes +# 9.22.0 (2026-08-03) + +This is a minor release introducing two flagship (experimental) features — **client-side caching** and **automatic pipelining** — alongside support for Redis 8.10, new commands, and a large batch of stability and parser-robustness fixes. It consolidates everything shipped in 9.22.0-beta.1, so the notes below cover the full 9.21.0 → 9.22.0 upgrade. + +⚠️ Two changes to be aware of when upgrading from 9.21.0: + +- **Default configuration values changed** ([#3918](https://github.com/redis/go-redis/pull/3918)): read/write timeouts, retry backoff, cluster state reload interval, and TCP keep-alive defaults are now aligned with the cross-SDK configuration proposal (see the highlight below). Explicitly configured values are unaffected. +- **`WaitAOF` return type corrected** ([#3888](https://github.com/redis/go-redis/pull/3888)): `WaitAOF` now returns `*IntSliceCmd`, matching the two-integer reply of `WAITAOF` (previously `*IntCmd`, which failed to parse the reply at runtime). Code referencing the old return type needs a one-line update. + +## 🚀 Highlights + +### Client-Side Caching (Experimental) + +The standalone `Client` gains server-assisted client-side caching built on RESP3 `CLIENT TRACKING`. Enable it by setting `ClientSideCacheConfig` in `Options` (or supply your own cache via `ClientSideCache` — e.g. to share one cache across clients). Cacheable read results are served from a local in-process cache and invalidated automatically when the server reports a change, cutting round trips for read-heavy workloads. + +The invalidation architecture is selected by `ClientSideCacheStrategy`; the default (and currently only) strategy is `CSCStrategySharedTracking`: one shared cache, every pool connection runs plain `CLIENT TRACKING ON`, and a background drainer applies buffered invalidations — portable (no BCAST) and consistent with the other Redis client libraries. Requirements and guardrails: RESP3 (`Protocol: 3`), standalone client, DB 0 only; commands that would change the connection identity (`SELECT`, `AUTH`, ...) are rejected while caching is enabled, and CSC is disabled when a credentials provider is set (fixed `Username`/`Password` work and are namespaced). See the README's [client-side caching section](README.md#client-side-caching) and the runnable [example](example/client-side-caching). + +**Experimental:** the API may change in a minor release. + +([#3941](https://github.com/redis/go-redis/pull/3941)) by [@ofekshenawa](https://github.com/ofekshenawa) + +### Automatic Pipelining (Experimental) + +`AutoPipeliner` is a background batcher that coalesces commands from many concurrent goroutines into Redis pipelines, multiplying throughput without any manual pipeline management. It comes in two faces, available on `Client` and `ClusterClient` (and configurable via `Options.AutoPipelineOptions` / `UniversalOptions.AutoPipelineOptions`): + +- **`AutoPipeline()`** — the blocking face: a drop-in `Cmdable` where each call blocks until executed, exactly like a plain client, while concurrent callers' commands batch together under the hood (measured locally over loopback: ~1M+ SET/sec vs ~100k unpipelined; indicative, not a guarantee). Per-goroutine command order is preserved. +- **`AsyncAutoPipeline()`** — the deferred face: command calls return immediately and every typed result accessor (`Val`/`Result`/`Err`/...) blocks until the command has executed. Submit a window of commands, then read the results, to keep pipelines deep (~2–3M SET/sec locally; indicative). + +`AutoPipelineOptions` controls batching: `MaxBatchSize` (soft target, default 200; the blocking face's preset uses 300), `MaxBatchBytes` (approximate payload cap so huge values flush as several bounded writes), `MaxFlushDelay` with optional `AdaptiveDelay` (delay scales down as the queue fills), and `MaxConcurrentBatches` (default 1 = a single ordered batch stream; raising it requires `Unordered: true`, so ordering is never lost by accident — `Validate()` rejects the combination otherwise). A usage tour and throughput comparison live in [`example/autopipeline`](example/autopipeline). + +**Experimental:** the API may change in a future release — pin your go-redis version if you adopt it. + +([#3942](https://github.com/redis/go-redis/pull/3942)) by [@ndyakov](https://github.com/ndyakov), with help from [@cxljs](https://github.com/cxljs) + +### Redis 8.10 Support + +This release adds support for **Redis 8.10**. The README's supported-versions list now includes Redis 8.10, and CI runs the full suite against the `redislabs/client-libs-test:8.10.0` image by default ([#3920](https://github.com/redis/go-redis/pull/3920), [#3940](https://github.com/redis/go-redis/pull/3940)). + +Coverage for the new commands and options that ship with Redis 8.10: + +- **`HIMPORT`** ([#3919](https://github.com/redis/go-redis/pull/3919)) — bulk hash import via server-side fieldsets, exposed as `HImportPrepare`, `HImportSet`, `HImportDiscard`, and `HImportDiscardAll`. Fieldsets are session state scoped to a single physical connection, which does not mix well with connection pooling — so the client keeps a versioned fieldset registry and lazily replays the `PREPARE` on whichever pooled connection executes a `SET` that needs it, at most once per connection, with no extra round trip (the `PREPARE` is injected into the same write as the `SET`). +- **`LMOVEM` / `BLMOVEM`** ([#3913](https://github.com/redis/go-redis/pull/3913)) — move multiple elements between lists in one call. +- **`SUNIONCARD` / `SDIFFCARD`** ([#3897](https://github.com/redis/go-redis/pull/3897)) — cardinality of set union/difference without materializing the result. +- **`XREAD` / `XREADGROUP` `MAXCOUNT` and `MAXSIZE`** ([#3898](https://github.com/redis/go-redis/pull/3898)) — bound how much data a stream read returns. +- **`TS.READ`** ([#3896](https://github.com/redis/go-redis/pull/3896)), **`TS.QUERYLABELS`** ([#3926](https://github.com/redis/go-redis/pull/3926)), **`TS.NRANGE` / `TS.NREVRANGE`** ([#3870](https://github.com/redis/go-redis/pull/3870)) with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)), and **`EXCLUDEEMPTY`** on `TS.MRANGE` / `TS.MREVRANGE` ([#3912](https://github.com/redis/go-redis/pull/3912)) — new time-series query surface. +- **`FT.ALIASLIST`** ([#3925](https://github.com/redis/go-redis/pull/3925)), **`COLLECT` reducer for `FT.AGGREGATE`** ([#3886](https://github.com/redis/go-redis/pull/3886)), **`RERANK` on HNSW vector fields in `FT.CREATE`** ([#3927](https://github.com/redis/go-redis/pull/3927)), and **`FT.HYBRID` timeout warnings** ([#3911](https://github.com/redis/go-redis/pull/3911)) — search coverage. + +### Cross-SDK Aligned Defaults + +Default configuration values now follow the cross-SDK configuration proposal shared by all Redis client libraries ([#3918](https://github.com/redis/go-redis/pull/3918)): + +| Setting | Old default | New default | +|---|---|---| +| `ReadTimeout` / `WriteTimeout` | 3s | 5s | +| Retry backoff (min/max) | 8ms / 512ms | 10ms / 1s | +| Cluster state reload interval | 10s | 60s | +| TCP keep-alive | 5min period | 30s idle / 5s interval / 3 probes (`net.KeepAliveConfig`) | + +Applications that set these values explicitly are unaffected; applications relying on the old defaults inherit the new ones. + +### Data-Race and Parser Hardening Sweep + +A systematic audit fixed data races across the client — hooks (`AddHook`, [#3868](https://github.com/redis/go-redis/pull/3868)), `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), cluster node slices ([#3861](https://github.com/redis/go-redis/pull/3861)), pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), maintenance notifications ([#3894](https://github.com/redis/go-redis/pull/3894), [#3872](https://github.com/redis/go-redis/pull/3872)), pool handoff ([#3876](https://github.com/redis/go-redis/pull/3876)), and `redisotel` ([#3881](https://github.com/redis/go-redis/pull/3881)) — and hardened the RESP parsers against malformed or unexpected replies: over-reads on nil replies ([#3874](https://github.com/redis/go-redis/pull/3874)), integer overflow when skipping map/attribute bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), unhashable RESP3 map keys ([#3873](https://github.com/redis/go-redis/pull/3873)), odd-length flat replies ([#3900](https://github.com/redis/go-redis/pull/3900)), mismatched declared array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), unexpected extra reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), and nil elements in numeric/bool slice replies ([#3922](https://github.com/redis/go-redis/pull/3922)). + +### PubSub `Receive` Hang Fix + +`PeekPushNotificationName` blocked until 36 bytes were buffered, so a short subscribe confirmation (channel name of six or fewer characters) on an otherwise idle connection hung `PubSub.Receive` forever — a regression introduced in 9.20.1 by [#3842](https://github.com/redis/go-redis/pull/3842). The peek now parses whatever is already buffered and only waits for one more byte when the frame prefix is valid but incomplete. Fixes [#3935](https://github.com/redis/go-redis/issues/3935). + +([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov) + +### Correct Cluster Transaction Retries + +The cluster transaction pipeline treated a `MULTI`...`EXEC` block as independently retryable commands, which could scatter a transaction across nodes or send malformed transactions on retry. Redirects (`MOVED`/`ASK`/`TRYAGAIN`) and aborts are now handled at the whole-transaction level, matching Redis transaction semantics: the transaction is re-routed and retried as a unit, never partially ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs). + +### Credential Redaction in Command Tracing + +`rediscmd.AppendCmd` — used by `redisotel` and `rediscensus` to render commands into span attributes — now redacts credential arguments as ``: `AUTH`, `HELLO ... AUTH`, `CONFIG SET` of `requirepass` / `masterauth` / TLS key passphrases, `ACL SETUSER` password rules, and `MIGRATE ... AUTH`/`AUTH2`. The client sends `HELLO ... AUTH` on every handshake and `AUTH` on every streaming-credentials rotation through the regular hook chain, so tracing hooks previously captured credentials even when the application never issued an auth command itself ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e). + +## ✨ New Features + +- **Client-side caching**: server-assisted caching for the standalone client via `ClientSideCacheConfig` / `ClientSideCache`, with the `CSCStrategySharedTracking` invalidation strategy ([#3941](https://github.com/redis/go-redis/pull/3941)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **Automatic pipelining**: `AutoPipeline()` (blocking) and `AsyncAutoPipeline()` (deferred results) on `Client` and `ClusterClient`, configured via `AutoPipelineOptions` ([#3942](https://github.com/redis/go-redis/pull/3942)) by [@ndyakov](https://github.com/ndyakov), with help from [@cxljs](https://github.com/cxljs) +- **`HIMPORT` command family**: `HImportPrepare` / `HImportSet` / `HImportDiscard` / `HImportDiscardAll` with lazy per-connection fieldset prepare replay ([#3919](https://github.com/redis/go-redis/pull/3919)) by [@ndyakov](https://github.com/ndyakov) +- **`LMOVEM` / `BLMOVEM`**: move multiple list elements in one call, with `COUNT` (up to N) or `EXACTLY` (all-or-nothing) semantics via `LMoveMArgs` ([#3913](https://github.com/redis/go-redis/pull/3913)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`SUnionCard` / `SDiffCard`**: cardinality of set union/difference ([#3897](https://github.com/redis/go-redis/pull/3897)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`XRead` / `XReadGroup` `MAXCOUNT` / `MAXSIZE`**: bound stream read responses by entry count or payload size ([#3898](https://github.com/redis/go-redis/pull/3898)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`TS.READ`**: read samples from a series starting at a given timestamp, with `TSReadEarliest` (`-`), `TSReadLatest` (`+`), and `TSReadNew` (`$`) sentinels ([#3896](https://github.com/redis/go-redis/pull/3896)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`TS.QUERYLABELS`**: query label names/values across time series ([#3926](https://github.com/redis/go-redis/pull/3926)) by [@ndyakov](https://github.com/ndyakov) +- **`TS.NRANGE` / `TS.NREVRANGE`**: range queries across multiple series ([#3870](https://github.com/redis/go-redis/pull/3870)) by [@ofekshenawa](https://github.com/ofekshenawa), with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)) by [@ndyakov](https://github.com/ndyakov) +- **`TS.MRANGE` / `TS.MREVRANGE` `EXCLUDEEMPTY`**: skip series with no samples in the result ([#3912](https://github.com/redis/go-redis/pull/3912)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.ALIASLIST`**: list all index aliases ([#3925](https://github.com/redis/go-redis/pull/3925)) by [@ndyakov](https://github.com/ndyakov) +- **`FT.AGGREGATE` `COLLECT` reducer**: collect grouped values into an array ([#3886](https://github.com/redis/go-redis/pull/3886)) by [@ndyakov](https://github.com/ndyakov) +- **`FT.CREATE` `RERANK`**: `RERANK` parameter on HNSW vector field definitions ([#3927](https://github.com/redis/go-redis/pull/3927)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.HYBRID` timeout warnings**: timeout warnings are now populated in hybrid search results ([#3911](https://github.com/redis/go-redis/pull/3911)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.HYBRID` KNN `SHARD_K_RATIO`** (Redis 8.8+): per-shard K ratio for KNN clauses ([#3841](https://github.com/redis/go-redis/pull/3841)) by [@ndyakov](https://github.com/ndyakov) + +## 🐛 Bug Fixes + +- **PubSub `Receive` hang**: peek push-notification names without demanding 36 buffered bytes, fixing a hang on short subscribe confirmations (fixes [#3935](https://github.com/redis/go-redis/issues/3935), regression from 9.20.1) ([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov) +- **Cluster transactions**: re-route the whole tx pipeline on redirect/abort instead of per-command ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs) +- **Credential leak in traces**: `rediscmd.AppendCmd` redacts credential arguments (`AUTH`, `HELLO ... AUTH`, `CONFIG SET` secret params, `ACL SETUSER` password rules, `MIGRATE AUTH`/`AUTH2`), so `redisotel` / `rediscensus` span attributes no longer contain passwords ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e) +- **`WaitAOF` return type**: returns `*IntSliceCmd` matching the two-integer `WAITAOF` reply ([#3888](https://github.com/redis/go-redis/pull/3888)) by [@CipherN9](https://github.com/CipherN9) +- **`Ring.Publish` routing**: publish to the shard that owns the topic instead of a round-robined one ([#3893](https://github.com/redis/go-redis/pull/3893)) by [@dkindel](https://github.com/dkindel) +- **Pool `OnRemove` hooks**: fire `OnRemove` on `putConn` eviction paths so removal hooks see every evicted connection ([#3932](https://github.com/redis/go-redis/pull/3932)) by [@cxljs](https://github.com/cxljs) +- **`UniversalClient` `InfoMap`**: added `InfoMap` to the `Cmdable` interface ([#3904](https://github.com/redis/go-redis/pull/3904)) by [@nazarli-shabnam](https://github.com/nazarli-shabnam) +- **`SlowLogGet` context**: pass the caller's context instead of a background one ([#3915](https://github.com/redis/go-redis/pull/3915)) by [@sonnemusk](https://github.com/sonnemusk) +- **`ModuleLoadex` nil config**: return an error instead of panicking on nil config ([#3916](https://github.com/redis/go-redis/pull/3916)) by [@sonnemusk](https://github.com/sonnemusk) +- **`ParseURL` IPv6 hosts**: keep single brackets for IPv6 hosts without a port ([#3882](https://github.com/redis/go-redis/pull/3882)) by [@sueun-dev](https://github.com/sueun-dev) +- **`ParseURL` durations**: treat unit durations `<= 0` as disabled ([#3866](https://github.com/redis/go-redis/pull/3866)) by [@sueun-dev](https://github.com/sueun-dev) +- **Nil `*uint8` encoding**: encode nil `*uint8` as `"0"` like other numeric pointers ([#3869](https://github.com/redis/go-redis/pull/3869)) by [@sueun-dev](https://github.com/sueun-dev) +- **`JSONSliceCmd` read errors**: return the read error from `readReply` instead of swallowing it ([#3903](https://github.com/redis/go-redis/pull/3903)) by [@saddamr3e](https://github.com/saddamr3e) +- **RESP parser hardening**: reconcile declared entry-array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), handle nil elements in int/uint/bool slice parsers ([#3922](https://github.com/redis/go-redis/pull/3922)), drain unexpected reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), reject odd-length flat replies in Z/KeyValue parsers ([#3900](https://github.com/redis/go-redis/pull/3900)), avoid int overflow when skipping map/attr bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), don't over-read nil replies in `Reader.Discard` ([#3874](https://github.com/redis/go-redis/pull/3874)) by [@saddamr3e](https://github.com/saddamr3e); reject unhashable keys in RESP3 map parsing ([#3873](https://github.com/redis/go-redis/pull/3873)) by [@iabdullah215](https://github.com/iabdullah215) +- **Data races**: hook state during `AddHook` ([#3868](https://github.com/redis/go-redis/pull/3868)), `onNewNode` during `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), shared masters/slaves slices in cluster ([#3861](https://github.com/redis/go-redis/pull/3861)), shared `opt.Addr` during pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), `clusterStateReloadCallback` in maintnotifications ([#3894](https://github.com/redis/go-redis/pull/3894)), conn reader in `isHealthyConn` during handoff ([#3876](https://github.com/redis/go-redis/pull/3876)) by [@saddamr3e](https://github.com/saddamr3e); handoff race window in maintnotifications ([#3872](https://github.com/redis/go-redis/pull/3872)) by [@ndyakov](https://github.com/ndyakov) +- **`redisotel`**: use `ObservableCounter` for cumulative pool stats ([#3914](https://github.com/redis/go-redis/pull/3914)) by [@Solaris-star](https://github.com/Solaris-star); avoid a data race on shared attributes during `MinIdleConns` warmup ([#3881](https://github.com/redis/go-redis/pull/3881)) by [@ndyakov](https://github.com/ndyakov) + +## 🧰 Maintenance + +- **Cross-SDK default alignment**: new defaults for timeouts, retry backoff, cluster state reload, and TCP keep-alive ([#3918](https://github.com/redis/go-redis/pull/3918)) by [@ndyakov](https://github.com/ndyakov) +- **CI on Redis 8.10**: 8.10 made the default test version ([#3920](https://github.com/redis/go-redis/pull/3920)) with version gating by major.minor ([#3908](https://github.com/redis/go-redis/pull/3908)) by [@ofekshenawa](https://github.com/ofekshenawa); the test stack now runs the GA `redislabs/client-libs-test:8.10.0` image and 8.8 was dropped from the CI matrix ([#3940](https://github.com/redis/go-redis/pull/3940)) +- **Type-safe atomics**: use typed `sync/atomic` value types ([#3860](https://github.com/redis/go-redis/pull/3860)) and remove the dead `assertUnstableCommand` RESP3 path ([#3928](https://github.com/redis/go-redis/pull/3928)) by [@cxljs](https://github.com/cxljs) +- **Docs**: clarify that `ExpireTime` / `PExpireTime` return Unix timestamps ([#3917](https://github.com/redis/go-redis/pull/3917)) by [@sonnemusk](https://github.com/sonnemusk); remove a duplicate example step ([#3875](https://github.com/redis/go-redis/pull/3875)) by [@andy-stark-redis](https://github.com/andy-stark-redis) + +## 👥 Contributors + +We'd like to thank all the contributors who worked on this release! + +[@andy-stark-redis](https://github.com/andy-stark-redis), [@CipherN9](https://github.com/CipherN9), [@cxljs](https://github.com/cxljs), [@dkindel](https://github.com/dkindel), [@iabdullah215](https://github.com/iabdullah215), [@nazarli-shabnam](https://github.com/nazarli-shabnam), [@ndyakov](https://github.com/ndyakov), [@ofekshenawa](https://github.com/ofekshenawa), [@saddamr3e](https://github.com/saddamr3e), [@Solaris-star](https://github.com/Solaris-star), [@sonnemusk](https://github.com/sonnemusk), [@sueun-dev](https://github.com/sueun-dev) + +--- + +**Full Changelog**: https://github.com/redis/go-redis/compare/v9.21.0...v9.22.0 + +# 9.22.0-beta.1 (2026-07-29) + +This is a **beta** release adding support for Redis 8.10, new commands, and a large batch of stability and parser-robustness fixes. The 9.22.0 GA release will follow once client-side caching and auto-pipelining are merged. + +⚠️ Two changes to be aware of when upgrading from 9.21.0: + +- **Default configuration values changed** ([#3918](https://github.com/redis/go-redis/pull/3918)): read/write timeouts, retry backoff, cluster state reload interval, and TCP keep-alive defaults are now aligned with the cross-SDK configuration proposal (see the highlight below). Explicitly configured values are unaffected. +- **`WaitAOF` return type corrected** ([#3888](https://github.com/redis/go-redis/pull/3888)): `WaitAOF` now returns `*IntSliceCmd`, matching the two-integer reply of `WAITAOF` (previously `*IntCmd`, which failed to parse the reply at runtime). Code referencing the old return type needs a one-line update. + +## 🚀 Highlights + +### Redis 8.10 Support + +This release adds support for **Redis 8.10**. The README's supported-versions list now includes Redis 8.10, and CI runs the full suite against the `redislabs/client-libs-test:8.10.0` image by default ([#3920](https://github.com/redis/go-redis/pull/3920), [#3940](https://github.com/redis/go-redis/pull/3940)). + +Coverage for the new commands and options that ship with Redis 8.10: + +- **`HIMPORT`** ([#3919](https://github.com/redis/go-redis/pull/3919)) — bulk hash import via server-side fieldsets, exposed as `HImportPrepare`, `HImportSet`, `HImportDiscard`, and `HImportDiscardAll`. Fieldsets are session state scoped to a single physical connection, which does not mix well with connection pooling — so the client keeps a versioned fieldset registry and lazily replays the `PREPARE` on whichever pooled connection executes a `SET` that needs it, at most once per connection, with no extra round trip (the `PREPARE` is injected into the same write as the `SET`). +- **`LMOVEM` / `BLMOVEM`** ([#3913](https://github.com/redis/go-redis/pull/3913)) — move multiple elements between lists in one call. +- **`SUNIONCARD` / `SDIFFCARD`** ([#3897](https://github.com/redis/go-redis/pull/3897)) — cardinality of set union/difference without materializing the result. +- **`XREAD` / `XREADGROUP` `MAXCOUNT` and `MAXSIZE`** ([#3898](https://github.com/redis/go-redis/pull/3898)) — bound how much data a stream read returns. +- **`TS.READ`** ([#3896](https://github.com/redis/go-redis/pull/3896)), **`TS.QUERYLABELS`** ([#3926](https://github.com/redis/go-redis/pull/3926)), **`TS.NRANGE` / `TS.NREVRANGE`** ([#3870](https://github.com/redis/go-redis/pull/3870)) with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)), and **`EXCLUDEEMPTY`** on `TS.MRANGE` / `TS.MREVRANGE` ([#3912](https://github.com/redis/go-redis/pull/3912)) — new time-series query surface. +- **`FT.ALIASLIST`** ([#3925](https://github.com/redis/go-redis/pull/3925)), **`COLLECT` reducer for `FT.AGGREGATE`** ([#3886](https://github.com/redis/go-redis/pull/3886)), **`RERANK` on HNSW vector fields in `FT.CREATE`** ([#3927](https://github.com/redis/go-redis/pull/3927)), and **`FT.HYBRID` timeout warnings** ([#3911](https://github.com/redis/go-redis/pull/3911)) — search coverage. + +### Cross-SDK Aligned Defaults + +Default configuration values now follow the cross-SDK configuration proposal shared by all Redis client libraries ([#3918](https://github.com/redis/go-redis/pull/3918)): + +| Setting | Old default | New default | +|---|---|---| +| `ReadTimeout` / `WriteTimeout` | 3s | 5s | +| Retry backoff (min/max) | 8ms / 512ms | 10ms / 1s | +| Cluster state reload interval | 10s | 60s | +| TCP keep-alive | 5min period | 30s idle / 5s interval / 3 probes (`net.KeepAliveConfig`) | + +Applications that set these values explicitly are unaffected; applications relying on the old defaults inherit the new ones. + +### Data-Race and Parser Hardening Sweep + +A systematic audit fixed data races across the client — hooks (`AddHook`, [#3868](https://github.com/redis/go-redis/pull/3868)), `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), cluster node slices ([#3861](https://github.com/redis/go-redis/pull/3861)), pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), maintenance notifications ([#3894](https://github.com/redis/go-redis/pull/3894), [#3872](https://github.com/redis/go-redis/pull/3872)), pool handoff ([#3876](https://github.com/redis/go-redis/pull/3876)), and `redisotel` ([#3881](https://github.com/redis/go-redis/pull/3881)) — and hardened the RESP parsers against malformed or unexpected replies: over-reads on nil replies ([#3874](https://github.com/redis/go-redis/pull/3874)), integer overflow when skipping map/attribute bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), unhashable RESP3 map keys ([#3873](https://github.com/redis/go-redis/pull/3873)), odd-length flat replies ([#3900](https://github.com/redis/go-redis/pull/3900)), mismatched declared array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), unexpected extra reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), and nil elements in numeric/bool slice replies ([#3922](https://github.com/redis/go-redis/pull/3922)). + +### PubSub `Receive` Hang Fix + +`PeekPushNotificationName` blocked until 36 bytes were buffered, so a short subscribe confirmation (channel name of six or fewer characters) on an otherwise idle connection hung `PubSub.Receive` forever — a regression introduced in 9.20.1 by [#3842](https://github.com/redis/go-redis/pull/3842). The peek now parses whatever is already buffered and only waits for one more byte when the frame prefix is valid but incomplete. Fixes [#3935](https://github.com/redis/go-redis/issues/3935). + +([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov) + +### Correct Cluster Transaction Retries + +The cluster transaction pipeline treated a `MULTI`...`EXEC` block as independently retryable commands, which could scatter a transaction across nodes or send malformed transactions on retry. Redirects (`MOVED`/`ASK`/`TRYAGAIN`) and aborts are now handled at the whole-transaction level, matching Redis transaction semantics: the transaction is re-routed and retried as a unit, never partially ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs). + +### Credential Redaction in Command Tracing + +`rediscmd.AppendCmd` — used by `redisotel` and `rediscensus` to render commands into span attributes — now redacts credential arguments as ``: `AUTH`, `HELLO ... AUTH`, `CONFIG SET` of `requirepass` / `masterauth` / TLS key passphrases, `ACL SETUSER` password rules, and `MIGRATE ... AUTH`/`AUTH2`. The client sends `HELLO ... AUTH` on every handshake and `AUTH` on every streaming-credentials rotation through the regular hook chain, so tracing hooks previously captured credentials even when the application never issued an auth command itself ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e). + +## ✨ New Features + +- **`HIMPORT` command family**: `HImportPrepare` / `HImportSet` / `HImportDiscard` / `HImportDiscardAll` with lazy per-connection fieldset prepare replay ([#3919](https://github.com/redis/go-redis/pull/3919)) by [@ndyakov](https://github.com/ndyakov) +- **`LMOVEM` / `BLMOVEM`**: move multiple list elements in one call, with `COUNT` (up to N) or `EXACTLY` (all-or-nothing) semantics via `LMoveMArgs` ([#3913](https://github.com/redis/go-redis/pull/3913)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`SUnionCard` / `SDiffCard`**: cardinality of set union/difference ([#3897](https://github.com/redis/go-redis/pull/3897)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`XRead` / `XReadGroup` `MAXCOUNT` / `MAXSIZE`**: bound stream read responses by entry count or payload size ([#3898](https://github.com/redis/go-redis/pull/3898)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`TS.READ`**: read samples from a series starting at a given timestamp, with `TSReadEarliest` (`-`), `TSReadLatest` (`+`), and `TSReadNew` (`$`) sentinels ([#3896](https://github.com/redis/go-redis/pull/3896)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`TS.QUERYLABELS`**: query label names/values across time series ([#3926](https://github.com/redis/go-redis/pull/3926)) by [@ndyakov](https://github.com/ndyakov) +- **`TS.NRANGE` / `TS.NREVRANGE`**: range queries across multiple series ([#3870](https://github.com/redis/go-redis/pull/3870)) by [@ofekshenawa](https://github.com/ofekshenawa), with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)) by [@ndyakov](https://github.com/ndyakov) +- **`TS.MRANGE` / `TS.MREVRANGE` `EXCLUDEEMPTY`**: skip series with no samples in the result ([#3912](https://github.com/redis/go-redis/pull/3912)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.ALIASLIST`**: list all index aliases ([#3925](https://github.com/redis/go-redis/pull/3925)) by [@ndyakov](https://github.com/ndyakov) +- **`FT.AGGREGATE` `COLLECT` reducer**: collect grouped values into an array ([#3886](https://github.com/redis/go-redis/pull/3886)) by [@ndyakov](https://github.com/ndyakov) +- **`FT.CREATE` `RERANK`**: `RERANK` parameter on HNSW vector field definitions ([#3927](https://github.com/redis/go-redis/pull/3927)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.HYBRID` timeout warnings**: timeout warnings are now populated in hybrid search results ([#3911](https://github.com/redis/go-redis/pull/3911)) by [@ofekshenawa](https://github.com/ofekshenawa) +- **`FT.HYBRID` KNN `SHARD_K_RATIO`** (Redis 8.8+): per-shard K ratio for KNN clauses ([#3841](https://github.com/redis/go-redis/pull/3841)) by [@ndyakov](https://github.com/ndyakov) + +## 🐛 Bug Fixes + +- **PubSub `Receive` hang**: peek push-notification names without demanding 36 buffered bytes, fixing a hang on short subscribe confirmations (fixes [#3935](https://github.com/redis/go-redis/issues/3935), regression from 9.20.1) ([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov) +- **Cluster transactions**: re-route the whole tx pipeline on redirect/abort instead of per-command ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs) +- **Credential leak in traces**: `rediscmd.AppendCmd` redacts credential arguments (`AUTH`, `HELLO ... AUTH`, `CONFIG SET` secret params, `ACL SETUSER` password rules, `MIGRATE AUTH`/`AUTH2`), so `redisotel` / `rediscensus` span attributes no longer contain passwords ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e) +- **`WaitAOF` return type**: returns `*IntSliceCmd` matching the two-integer `WAITAOF` reply ([#3888](https://github.com/redis/go-redis/pull/3888)) by [@CipherN9](https://github.com/CipherN9) +- **`Ring.Publish` routing**: publish to the shard that owns the topic instead of a round-robined one ([#3893](https://github.com/redis/go-redis/pull/3893)) by [@dkindel](https://github.com/dkindel) +- **Pool `OnRemove` hooks**: fire `OnRemove` on `putConn` eviction paths so removal hooks see every evicted connection ([#3932](https://github.com/redis/go-redis/pull/3932)) by [@cxljs](https://github.com/cxljs) +- **`UniversalClient` `InfoMap`**: added `InfoMap` to the `Cmdable` interface ([#3904](https://github.com/redis/go-redis/pull/3904)) by [@nazarli-shabnam](https://github.com/nazarli-shabnam) +- **`SlowLogGet` context**: pass the caller's context instead of a background one ([#3915](https://github.com/redis/go-redis/pull/3915)) by [@sonnemusk](https://github.com/sonnemusk) +- **`ModuleLoadex` nil config**: return an error instead of panicking on nil config ([#3916](https://github.com/redis/go-redis/pull/3916)) by [@sonnemusk](https://github.com/sonnemusk) +- **`ParseURL` IPv6 hosts**: keep single brackets for IPv6 hosts without a port ([#3882](https://github.com/redis/go-redis/pull/3882)) by [@sueun-dev](https://github.com/sueun-dev) +- **`ParseURL` durations**: treat unit durations `<= 0` as disabled ([#3866](https://github.com/redis/go-redis/pull/3866)) by [@sueun-dev](https://github.com/sueun-dev) +- **Nil `*uint8` encoding**: encode nil `*uint8` as `"0"` like other numeric pointers ([#3869](https://github.com/redis/go-redis/pull/3869)) by [@sueun-dev](https://github.com/sueun-dev) +- **`JSONSliceCmd` read errors**: return the read error from `readReply` instead of swallowing it ([#3903](https://github.com/redis/go-redis/pull/3903)) by [@saddamr3e](https://github.com/saddamr3e) +- **RESP parser hardening**: reconcile declared entry-array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), handle nil elements in int/uint/bool slice parsers ([#3922](https://github.com/redis/go-redis/pull/3922)), drain unexpected reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), reject odd-length flat replies in Z/KeyValue parsers ([#3900](https://github.com/redis/go-redis/pull/3900)), avoid int overflow when skipping map/attr bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), don't over-read nil replies in `Reader.Discard` ([#3874](https://github.com/redis/go-redis/pull/3874)) by [@saddamr3e](https://github.com/saddamr3e); reject unhashable keys in RESP3 map parsing ([#3873](https://github.com/redis/go-redis/pull/3873)) by [@iabdullah215](https://github.com/iabdullah215) +- **Data races**: hook state during `AddHook` ([#3868](https://github.com/redis/go-redis/pull/3868)), `onNewNode` during `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), shared masters/slaves slices in cluster ([#3861](https://github.com/redis/go-redis/pull/3861)), shared `opt.Addr` during pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), `clusterStateReloadCallback` in maintnotifications ([#3894](https://github.com/redis/go-redis/pull/3894)), conn reader in `isHealthyConn` during handoff ([#3876](https://github.com/redis/go-redis/pull/3876)) by [@saddamr3e](https://github.com/saddamr3e); handoff race window in maintnotifications ([#3872](https://github.com/redis/go-redis/pull/3872)) by [@ndyakov](https://github.com/ndyakov) +- **`redisotel`**: use `ObservableCounter` for cumulative pool stats ([#3914](https://github.com/redis/go-redis/pull/3914)) by [@Solaris-star](https://github.com/Solaris-star); avoid a data race on shared attributes during `MinIdleConns` warmup ([#3881](https://github.com/redis/go-redis/pull/3881)) by [@ndyakov](https://github.com/ndyakov) + +## 🧰 Maintenance + +- **Cross-SDK default alignment**: new defaults for timeouts, retry backoff, cluster state reload, and TCP keep-alive ([#3918](https://github.com/redis/go-redis/pull/3918)) by [@ndyakov](https://github.com/ndyakov) +- **CI on Redis 8.10**: 8.10 made the default test version ([#3920](https://github.com/redis/go-redis/pull/3920)) with version gating by major.minor ([#3908](https://github.com/redis/go-redis/pull/3908)) by [@ofekshenawa](https://github.com/ofekshenawa); the test stack now runs the GA `redislabs/client-libs-test:8.10.0` image and 8.8 was dropped from the CI matrix ([#3940](https://github.com/redis/go-redis/pull/3940)) +- **Type-safe atomics**: use typed `sync/atomic` value types ([#3860](https://github.com/redis/go-redis/pull/3860)) and remove the dead `assertUnstableCommand` RESP3 path ([#3928](https://github.com/redis/go-redis/pull/3928)) by [@cxljs](https://github.com/cxljs) +- **Docs**: clarify that `ExpireTime` / `PExpireTime` return Unix timestamps ([#3917](https://github.com/redis/go-redis/pull/3917)) by [@sonnemusk](https://github.com/sonnemusk); remove a duplicate example step ([#3875](https://github.com/redis/go-redis/pull/3875)) by [@andy-stark-redis](https://github.com/andy-stark-redis) + +## 👥 Contributors + +We'd like to thank all the contributors who worked on this release! + +[@andy-stark-redis](https://github.com/andy-stark-redis), [@CipherN9](https://github.com/CipherN9), [@cxljs](https://github.com/cxljs), [@dkindel](https://github.com/dkindel), [@iabdullah215](https://github.com/iabdullah215), [@nazarli-shabnam](https://github.com/nazarli-shabnam), [@ndyakov](https://github.com/ndyakov), [@ofekshenawa](https://github.com/ofekshenawa), [@saddamr3e](https://github.com/saddamr3e), [@Solaris-star](https://github.com/Solaris-star), [@sonnemusk](https://github.com/sonnemusk), [@sueun-dev](https://github.com/sueun-dev) + +--- + +**Full Changelog**: https://github.com/redis/go-redis/compare/v9.21.0...v9.22.0-beta.1 + # 9.21.0 (2026-06-18) This is a minor release adding new features and bug fixes. There are no breaking changes; upgrading from 9.20.x is a drop-in replacement. diff --git a/backend/vendor/github.com/redis/go-redis/v9/autopipeline.go b/backend/vendor/github.com/redis/go-redis/v9/autopipeline.go new file mode 100644 index 00000000..42ac8d50 --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/autopipeline.go @@ -0,0 +1,2609 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "io" + "runtime" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sys/cpu" + + "github.com/redis/go-redis/v9/internal" +) + +// AutoPipelineOptions configures the autopipelining behavior. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +type AutoPipelineOptions struct { + // MaxBatchSize is the target batch size: the accumulator stops waiting for + // more commands once the shard queue reaches it, so a batch flushes promptly + // instead of lingering. It is a soft threshold, not a hard cap — under heavy + // concurrent enqueue (or while a flush waits on the concurrency semaphore) the + // queue can grow past it and execute as a single larger pipeline, which is + // safe and simply yields a deeper pipeline. + // Default: 200 (the blocking face's no-options preset, + // DefaultBlockingAutoPipelineOptions, uses 300). + MaxBatchSize int + + // MaxBatchBytes, when > 0, caps a batch by APPROXIMATE payload volume: the + // accumulator stops waiting once the queued commands' argument bytes reach + // it, so many large values flush as several bounded writes instead of one + // huge burst (300 x 64KiB is ~19MB written down one connection before any + // reply is read — enough to stall a constrained link past its write + // deadline). Like MaxBatchSize it is a soft threshold, not a hard cap. + // The estimate counts string/[]byte argument lengths plus a small + // per-argument overhead. Default: 0 (no byte cap). + MaxBatchBytes int + + // MaxConcurrentBatches is the maximum number of pipeline batches that may + // execute concurrently. + // + // Default: 1, which gives a single ordered command stream — batches execute + // serially in submit order, so even a windowed caller (submit many, read + // later) sees strict ordering, while still reaching high throughput via deep + // pipelines (~3M ops/sec locally). + // + // Setting this above 1 runs batches in parallel for maximum throughput, but + // commands then have NO guaranteed execution order. Because that trades away + // ordering, it is only allowed together with Unordered: true — otherwise the + // configuration is rejected (see Validate). This makes the trade-off + // explicit: you cannot accidentally lose ordering by raising concurrency. + MaxConcurrentBatches int + + // Unordered must be set to true to allow MaxConcurrentBatches > 1. It is the + // caller's explicit acknowledgement that parallel batch execution gives up + // command ordering in exchange for throughput. With the default (false), + // MaxConcurrentBatches is forced to 1 (an ordered stream) and any value > 1 + // is a configuration error. + Unordered bool + + // contentSharded is set internally by cluster wiring when commands are + // routed to shards by content (slot), so same-key commands always share a + // shard and per-key order holds even with several shards. It exempts that + // wiring from the NumShards ordering check in newAutoPipeliner. Never set + // by users (unexported). + contentSharded bool + + // NumShards is the number of independent queue+flusher shards the + // autopipeliner runs. 0 (the default) means auto: a single shard, which + // funnels every caller into one queue so batches stay deep — measured + // throughput and latency are best with one shard even under heavy + // goroutine concurrency. Cluster clients default to several slot-routed + // shards instead, so commands for different nodes queue independently + // (per-key order still holds: a key's slot always maps to the same + // shard). Raising NumShards splits the queue: it reduces enqueue-mutex + // contention but fragments batches, which usually costs far more than the + // contention saves. Every shard always has at least one concurrency + // permit, so the effective global batch concurrency is + // max(NumShards, MaxConcurrentBatches) — and because shards flush + // concurrently, NumShards > 1 on the deferred (async) face requires + // Unordered: true (construction fails otherwise). + NumShards int + + // MaxFlushDelay is the maximum delay after flushing before checking for more commands. + // A small delay (e.g., 100μs) can significantly reduce CPU usage by allowing + // more commands to batch together, at the cost of slightly higher latency. + // + // Trade-off: + // - 0 (default): Lowest latency, higher CPU usage + // - 100μs: Balanced (recommended for most workloads) + // - 500μs: Lower CPU usage, higher latency + // + // Based on benchmarks, 100μs can reduce CPU usage by 50% + // while adding only ~100μs average latency per command. + // Default: 0, meaning the flusher applies no coalescing wait — it flushes + // each batch as soon as the queue is ready and lets in-flight backpressure + // coalesce concurrent callers (see accumulateBatch). Set a value here to add + // an explicit accumulation window, trading latency for larger batches / less + // CPU as described above. + MaxFlushDelay time.Duration + + // AdaptiveDelay enables smart delay calculation based on queue fill level. + // When enabled, the delay is automatically adjusted: + // - Queue ≥75% full: No delay (flush immediately to prevent overflow) + // - Queue ≥50% full: 25% of MaxFlushDelay (queue filling up) + // - Queue ≥25% full: 50% of MaxFlushDelay (moderate load) + // - Queue <25% full: 100% of MaxFlushDelay (low load, maximize batching) + // + // This provides automatic adaptation to varying load patterns without + // manual tuning. Uses integer-only arithmetic for optimal performance. + // Default: false (use fixed MaxFlushDelay) + AdaptiveDelay bool +} + +// autoPipelinePermitBackstop bounds how long a flush waits for a concurrency +// permit when all are busy. It is only a safety net against a wedged semaphore: +// every permit holder releases it (via defer) and each batch Exec is itself +// bounded by the connection's read/write timeout, so in normal operation a +// permit frees long before this. It is set well above the default ReadTimeout +// and a maintnotifications relaxed window so a legitimately slow in-flight batch +// never makes waiters fail spuriously. The wait deliberately does NOT end on +// Close: commands taken from the queue were already accepted, and Close's +// contract is to flush them (it waits via wg/batchWg), so permit waits run on +// a background context bounded only by this backstop. +const autoPipelinePermitBackstop = 30 * time.Second + +// autoPipelineCloseBackstop bounds Close's wait for in-flight dispatches. It +// deliberately carries the same value as the permit backstop but its OWN name: +// the two answer different questions, and this one may want tuning on its own. +// +// Why it is generous rather than snappy: the bound is only ever REACHED when a +// dispatch cannot end by itself — a blocking command with no timeout, or a +// stalled read with ReadTimeout disabled. In every other configuration the +// read timeout ends the dispatch and Close returns the moment it does, well +// under this value. A tighter bound would not speed up healthy shutdowns; it +// would instead make Close report failure while legitimate work is still +// finishing (a large final batch, or a maintnotifications relaxed window +// during a failover), turning a correct slow drain into a spurious error. +const autoPipelineCloseBackstop = 30 * time.Second + +// numAutoPipelineShards is the shard-count default used by CLUSTER wiring, +// where commands are routed to shards by slot so different nodes' batches +// queue independently (every shard keeps at least one concurrency permit, so +// several shards can flush to their nodes in parallel regardless of +// MaxConcurrentBatches). It is NOT used for standalone clients: those default +// to one shard (see newAutoPipeliner), because a single deep queue pipelines +// far better than a fragmented one. Deliberately NOT derived from +// MaxConcurrentBatches — coupling shard count to the permit budget silently +// collapsed cluster slot routing to a single shard at the default budget. +func numAutoPipelineShards() int { + n := runtime.GOMAXPROCS(0) + if n < 1 { + n = 1 + } + const maxShards = 16 + if n > maxShards { + n = maxShards + } + return n +} + +// DefaultAutoPipelineOptions returns the default autopipelining configuration. +// +// The default is ordered: MaxConcurrentBatches is 1, so batches execute +// serially in submit order (a single ordered command stream) while still +// reaching high throughput via deep pipelines when callers submit in windows. +// To trade ordering for parallel-batch throughput, set MaxConcurrentBatches > 1 +// together with Unordered: true. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func DefaultAutoPipelineOptions() *AutoPipelineOptions { + return &AutoPipelineOptions{ + MaxBatchSize: 200, + MaxConcurrentBatches: 1, // ordered by default + MaxFlushDelay: 0, // lowest latency; no coalescing wait (batch via in-flight backpressure) + } +} + +// DefaultBlockingAutoPipelineOptions returns the default config for the +// blocking face (Client.AutoPipeline). It uses a single ordered batch stream +// (MaxConcurrentBatches: 1). Counterintuitively this maximizes throughput AND +// minimizes latency for the blocking face: with one batch in flight, callers whose +// commands return while it executes re-enqueue and flush together as the next +// batch, so batches stay deep (a near-continuous, double-buffered pipeline), +// while a lone caller flushes promptly in a single round-trip (no coalescing +// wait — see accumulateBatch). More parallel permits (MaxConcurrentBatches>1) do the +// opposite: each command finds a free permit and flushes on its own before +// others accumulate, collapsing batch size — and throughput — toward one command +// per round-trip while latency rises. For maximum throughput use the async face +// (AsyncAutoPipeline) with a window of in-flight commands (inflight>1); it keeps +// MaxConcurrentBatches: 1 as well. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func DefaultBlockingAutoPipelineOptions() *AutoPipelineOptions { + return &AutoPipelineOptions{ + MaxBatchSize: 300, + MaxConcurrentBatches: 1, + } +} + +// Validate reports whether the configuration is self-consistent. It returns an +// error if MaxConcurrentBatches > 1 without Unordered: true — raising +// concurrency gives up command ordering, so the caller must opt in explicitly. +// +// Validate()==nil does not guarantee construction succeeds: rules that need +// the face (e.g. NumShards>1 requires Unordered on the deferred face) are +// enforced by the AutoPipeline/AsyncAutoPipeline getters. Note also that +// Options.AutoPipelineOptions is validated lazily — on the first getter +// call, not in NewClient. +func (cfg *AutoPipelineOptions) Validate() error { + if cfg.MaxConcurrentBatches > 1 && !cfg.Unordered { + return fmt.Errorf("redis: AutoPipelineOptions.MaxConcurrentBatches=%d requires Unordered:true "+ + "(parallel batches do not preserve command ordering); set Unordered:true to allow it, "+ + "or keep MaxConcurrentBatches=1 for an ordered stream", cfg.MaxConcurrentBatches) + } + // Reject obviously-wrong negatives so a typo surfaces at construction rather + // than being silently coerced to a default. Zero is allowed and means "use + // the default" (MaxBatchSize) or "no delay" (MaxFlushDelay). + if cfg.MaxBatchSize < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.MaxBatchSize=%d must be >= 0", cfg.MaxBatchSize) + } + if cfg.MaxBatchBytes < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.MaxBatchBytes=%d must be >= 0", cfg.MaxBatchBytes) + } + if cfg.MaxConcurrentBatches < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.MaxConcurrentBatches=%d must be >= 0", cfg.MaxConcurrentBatches) + } + if cfg.MaxFlushDelay < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.MaxFlushDelay=%s must be >= 0", cfg.MaxFlushDelay) + } + if cfg.NumShards < 0 { + return fmt.Errorf("redis: AutoPipelineOptions.NumShards=%d must be >= 0", cfg.NumShards) + } + if cfg.AdaptiveDelay && cfg.MaxFlushDelay <= 0 { + return fmt.Errorf("redis: AutoPipelineOptions.AdaptiveDelay requires MaxFlushDelay > 0 " + + "(adaptive delay scales MaxFlushDelay by queue fill; with no MaxFlushDelay it would " + + "silently disable batch accumulation entirely)") + } + return nil +} + +// cmdableClient is an interface for clients that support pipelining. +// Both Client and ClusterClient implement this interface. It embeds +// UniversalClient (Cmdable + Process + Do + AddHook + Watch + Subscribe... + +// Close + PoolStats) so the AutoPipeliner can delegate the non-batched surface +// back to the underlying client and itself satisfy UniversalClient. +type cmdableClient interface { + UniversalClient + // processPipelineHook is the hook-wrapped []Cmder pipeline entry — the same + // method Pipeline.Exec is wired to (see Client.Pipeline). The flusher + // dispatches drained batches through it directly, skipping the per-batch + // Pipeline construction; hooks/OTel see the identical call. + processPipelineHook(ctx context.Context, cmds []Cmder) error + // The async faces additionally dispatch through withProcessPipelineHook / + // withProcessHook with the base processors as the innermost, so the batch + // can be completed UNDER the user hooks (results ready the moment exec + // returns, before hooks unwind). Both *Client and *ClusterClient satisfy + // these via hooksMixin and their base processors. + withProcessPipelineHook(ctx context.Context, cmds []Cmder, hook ProcessPipelineHook) error + hookCount() int + withProcessHook(ctx context.Context, cmd Cmder, hook ProcessHook) error + processPipeline(ctx context.Context, cmds []Cmder) error + process(ctx context.Context, cmd Cmder) error +} + +// apBatch is the completion signal shared by every command flushed together. +// Its done channel is closed exactly once, when the batch's pipeline has +// executed. Closing one channel wakes all waiters in a single operation, +// instead of doing one buffered-channel send per command — under high +// concurrency the per-command sends dominated CPU (channel-lock contention and +// one goroutine wake-up apiece). +type apBatch struct { + done chan struct{} + // closed makes close() idempotent: on the async faces the dispatch closes + // the batch at the innermost exec seam (under the user hooks, so a hook + // reading a result after next() does not block on a channel its own + // goroutine closes — the #3867 deadlock), while the flusher keeps its + // deferred close as a panic backstop. Whichever runs first wins. + closed atomic.Bool + // dispGid is the goroutine id of the dispatcher while the batch is inside + // the hook chain (0 otherwise). await() consults it before blocking so a + // hook on the dispatch goroutine reading a result BEFORE next() gets the + // not-yet-executed view — what a plain pipeline hook sees — instead of a + // self-deadlock. + dispGid atomic.Int64 + // nodeGids registers cluster per-node executor goroutines: the cluster + // pipeline fans a batch out to one goroutine per node, and each runs the + // NODE client's own hook chain (OnNewNode hooks — redisotel's tracing + // lives there), which the single dispGid slot cannot vouch for. A node + // hook reading a result there would block on a batch that completes only + // after its own return — reproduced as a permanent wedge with a + // rediscmd-shaped Err() peek. Guarded by nodeMu; entered/left once per + // node call, consulted only on the guards' slow path (done still open). + nodeMu sync.Mutex + nodeGids []int64 + // nodeCount mirrors len(nodeGids) so isExecutorGoroutine's fast path can + // skip the goroutine-id parse and the mutex entirely when nobody is + // registered — which is every standalone batch, always, and a cluster + // batch outside its node fan-out window. + nodeCount atomic.Int32 +} + +// enterNodeDispatch registers the calling goroutine as an executor of this +// batch for the duration of a cluster node call; the returned func +// unregisters it. Registered goroutines get the same treatment as the +// dispatcher in the accessor guards: result reads return the current view +// instead of self-deadlocking on the batch's own completion signal. +func (b *apBatch) enterNodeDispatch() func() { + gid := curGoroutineID() + b.nodeMu.Lock() + b.nodeGids = append(b.nodeGids, gid) + b.nodeCount.Store(int32(len(b.nodeGids))) + b.nodeMu.Unlock() + return func() { + b.nodeMu.Lock() + for i, g := range b.nodeGids { + if g == gid { + b.nodeGids[i] = b.nodeGids[len(b.nodeGids)-1] + b.nodeGids = b.nodeGids[:len(b.nodeGids)-1] + break + } + } + b.nodeCount.Store(int32(len(b.nodeGids))) + b.nodeMu.Unlock() + } +} + +// isExecutorGoroutine reports whether the CALLING goroutine is currently +// executing this batch: the flusher/dispatch goroutine or a registered +// cluster node executor. The no-executor fast path (dispGid unset and no +// node executors) is two atomic loads — no goroutine-id parse, no lock. That +// laziness is load-bearing: every blocking-face command and every pre-done +// future passes here once per wait, and an earlier revision that parsed the +// goroutine id and took the mutex unconditionally cost the blocking face 6x +// of its throughput (measured 830k -> 138k ops/sec on a loopback bench). +func (b *apBatch) isExecutorGoroutine() bool { + disp := b.dispGid.Load() + if disp == 0 && b.nodeCount.Load() == 0 { + return false + } + gid := curGoroutineID() + if disp != 0 && disp == gid { + return true + } + if b.nodeCount.Load() == 0 { + return false + } + b.nodeMu.Lock() + defer b.nodeMu.Unlock() + for _, g := range b.nodeGids { + if g == gid { + return true + } + } + return false +} + +// noopUnregister is registerBatchExecutors' zero-batch result, shared so the +// plain-pipeline path stays allocation-free. +var noopUnregister = func() {} + +// registerBatchExecutors marks the calling goroutine as an executor of every +// deferred-face batch among cmds (plain pipeline commands carry none) and +// returns the combined unregister. The cluster pipeline calls it around each +// node's hook chain. +func registerBatchExecutors(cmds []Cmder) func() { + var undo []func() + var seenFirst *apBatch + var seenMore map[*apBatch]struct{} + for _, cmd := range cmds { + bc, ok := cmd.(interface{ readyBatch() *apBatch }) + if !ok { + continue + } + b := bc.readyBatch() + if b == nil || b == seenFirst { + continue + } + if seenFirst == nil { + seenFirst = b + } else { + if seenMore == nil { + seenMore = make(map[*apBatch]struct{}, 2) + } + if _, dup := seenMore[b]; dup { + continue + } + seenMore[b] = struct{}{} + } + undo = append(undo, b.enterNodeDispatch()) + } + if len(undo) == 0 { + return noopUnregister + } + return func() { + for _, u := range undo { + u() + } + } +} + +func newAPBatch() *apBatch { return &apBatch{done: make(chan struct{})} } + +// close completes the batch exactly once, waking every waiter. +func (b *apBatch) close() { + if b.closed.CompareAndSwap(false, true) { + close(b.done) + } +} + +// curGoroutineID parses the goroutine id from runtime.Stack's header +// ("goroutine 123 ["). Called only on paths already paying a dispatch or an +// about-to-block round-trip wait — never on await()'s fast path — so the +// microsecond-scale stack read is noise against the batch RTT. +// armSelfDeadlockGuard reports whether async dispatch should stamp the +// dispatcher's goroutine id on the batches (see apBatch.dispGid) — the +// mechanism that lets a hook on the dispatch goroutine read a command +// without deadlocking on a batch only that goroutine completes: before +// next() it sees the not-yet-executed view, after next() the populated +// results (batches complete only when the whole chain has returned). Armed +// when user hooks exist — without hooks nothing can read a command inside +// the chain — and always on cluster clients, whose node clients may carry +// their own hooks (OnNewNode + AddHook, the redisotel pattern) that +// hookCount() cannot see. NOTE: node-level hooks run on node-worker +// goroutines the gid guard cannot identify, so they must not read command +// results on the async face; the same applies to a goroutine a hook spawns +// and joins before returning. A hook added concurrently with an in-flight +// dispatch misses the guard for that one batch. The guard covers result +// READS only: a hook that ISSUES a command on the same AutoPipeliner and +// synchronously waits for it cannot be saved — the nested command needs the +// dispatch slot the hook chain is holding, and the engine recovers only by +// failing the flush after the permit backstops (see +// autoPipelinePermitBackstop) expire. +func (ap *AutoPipeliner) armSelfDeadlockGuard() bool { + return ap.pipeliner.hookCount() > 0 || ap.config.contentSharded +} + +func curGoroutineID() int64 { + var buf [64]byte + n := runtime.Stack(buf[:], false) + const skip = len("goroutine ") + var id int64 + for _, c := range buf[skip:n] { + if c < '0' || c > '9' { + break + } + id = id*10 + int64(c-'0') + } + return id +} + +// The shard queue stores bare Cmders. The batch a command waits on is the +// shard's curBatch at enqueue time — read once to wire the command's ready +// channel and never needed per-command afterward (the flusher closes the one +// shared batch). Storing []Cmder removes a per-command wrapper allocation. + +var queueSlicePool = sync.Pool{ + New: func() interface{} { s := make([]Cmder, 0, 100); return &s }, +} + +func getQueueSlice(capacity int) []Cmder { + slice := (*queueSlicePool.Get().(*[]Cmder))[:0] + if cap(slice) < capacity { + queueSlicePool.Put(&slice) + return make([]Cmder, 0, capacity) + } + return slice +} + +func putQueueSlice(slice []Cmder) { + if cap(slice) <= 1000 { + // Zero only the used prefix: elements beyond len are already nil — + // slices enter the pool fully zeroed (here) and are only appended to + // afterwards, so the tail invariant holds. Zeroing the whole capacity + // memclr'd up to 8 KB per flush for small batches on large recycled + // arrays. + for i := range slice { + slice[i] = nil + } + queueSlicePool.Put(&slice) + } +} + +// AutoPipeliner automatically batches commands and executes them in pipelines. +// It's safe for concurrent use by multiple goroutines. +// +// AutoPipeliner works by collecting commands from multiple goroutines into a +// shared queue and flushing them as one Redis pipeline when the batch reaches +// MaxBatchSize or a configured coalescing window (MaxFlushDelay) elapses. By +// default there is no window: each batch flushes as soon as the queue is ready +// and concurrent callers coalesce via in-flight backpressure, so a lone command +// flushes in a single round-trip while batches stay deep under load. +// +// This provides significant performance improvements for workloads with many +// concurrent small operations, as it reduces the number of network round-trips. +// +// AutoPipeliner implements the Cmdable interface, so you can use it like a +// regular client. Prefer the typed methods (Set, Get, ...); Do runs OUTSIDE +// the pipeline on a normal connection (see Do). +// AutoPipeline / AsyncAutoPipeline return an error for an invalid config, so check it once: +// +// ap, err := client.AutoPipeline() +// if err != nil { +// return err +// } +// ap.Set(ctx, "key", "value", 0) +// ap.Get(ctx, "key") +// ap.Close() +// +// Per-command contexts: a command is batched and executed on the AutoPipeliner's +// own long-lived context, NOT the context passed to the command. A per-command +// deadline or cancellation is therefore not honored once the command is queued +// (this is deliberate — a per-batch timer per command would cost a goroutine +// each). Use a plain client for commands that need their own deadline. +// The one exception is a blocking command (readTimeout() != nil, e.g. BLPOP): +// it is never batched and runs directly on the caller's context, which is +// honored as usual. +// +// Retries: like any pipeline, a batch that fails on a network error is retried +// as a whole (up to Options.MaxRetries). If the connection drops after the +// server executed part of the batch, non-idempotent commands (INCR, LPUSH, ...) +// may execute twice. Run commands that must not be retransmitted on a plain +// client, or set MaxRetries: -1. +// +// Lifetime: AutoPipeline() returns a single, client-owned instance shared by all +// callers. Close()ing it stops the shared pipeliner for everyone; a later +// AutoPipeline() call on the client builds a fresh one. Closing the CLIENT also +// stops it, but permanently: the getters then return ErrClosed. +// +// Formatting: String()/%v on a command issued by the deferred face WAITS for +// execution, exactly like Err()/Val()/Result() — formatting reads the result +// fields, and reading them unsynchronized would race the dispatcher populating +// them. The one exception is a hook formatting a command from the batch's own +// dispatch goroutine: that returns the not-yet-executed view instead of +// self-deadlocking. Use Name()/Args() if you need to log a submission without +// waiting for it. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +type AutoPipeliner struct { + cmdable // Embed cmdable to get all Redis command methods + + pipeliner cmdableClient + config *AutoPipelineOptions + // blocking selects how the typed command surface (Set, Get, ...) behaves: + // when true the command call itself blocks until the command has executed + // (drop-in, synchronous shape); when false the call returns immediately and + // the result accessors (Val/Result/Err) block. See AutoPipeline (blocking) + // vs AsyncAutoPipeline (deferred). + blocking bool + + // Sharded command queues. Each shard has its own queue, mutex and flusher + // goroutine, so enqueues from many goroutines spread across shards instead + // of all contending on a single mutex and being drained by a single + // flusher. Commands are assigned to shards round-robin; per-goroutine + // ordering is still guaranteed because Do blocks for each command's result + // before issuing the next one. + shards []*apShard + next atomic.Uint32 // round-robin shard selector + // shardFn, when set, picks a command's shard from its content (cluster mode + // sets it to route by slot so all commands for one node land in the same + // shard's batch — keeping per-node pipelines deep instead of splitting every + // batch across nodes). When nil, commands are assigned round-robin. + shardFn func(Cmder) int + + // preflight, when set, can reject a command at submit time, before it is + // enqueued or dispatched (cluster mode refuses fan-out-policy commands + // that cannot ride a pipeline, so one caller's command cannot poison a + // merged batch). The returned error is set on the command. + preflight func(ctx context.Context, cmd Cmder) error + + // mustDivert, when set, forces a command off the batching path even though + // it is otherwise batchable — cluster mode uses it for commands whose + // routing is NOT slot-derived (ReqSpecial, e.g. FT.CURSOR READ, which is + // sticky to the node that owns the cursor). Batched, mapCmdsByNode would + // route them by slot and reach the wrong shard; diverted, they go through + // Client/ClusterClient.Process and keep their special routing. + mustDivert func(ctx context.Context, cmd Cmder) bool + + // sharedClosed, when non-nil, is the owning client's pool-set closed flag + // (shared across WithTimeout clones). The getters refuse to build a fresh + // pipeliner once it is set; this reference makes an ALREADY-built + // pipeliner refuse new work too — without it, a clone's Close would leave + // a cached pipeliner accepting enqueues against closed pools, failing + // them one dispatch at a time instead of with ErrClosed at submit. + sharedClosed *atomic.Bool + + // expectedArrivals counts how many commands the engine expects to arrive + // at any moment: a completed batch of N≥2 commands wakes its N waiters + // together, and in a closed loop each immediately submits its next command + // — so completion announces N expected arrivals, and every enqueue accounts + // for one. The default coalescing wait (awaitExpectedArrivals) holds the + // flusher while arrivals are still expected, so the whole wakeup wave + // flushes as one deep pipeline — an exact count, not a smoothed estimate, + // which cannot ratchet into fragmentation. Single-command batches announce + // nothing, so a lone caller and open-loop traffic never wait. May + // transiently go negative (arrivals nobody announced); readers clamp to + // zero. Pipeliner-global, not per-shard: cluster routing may land a + // follow-up on a different shard than the batch that woke its caller. + expectedArrivals atomic.Int64 + + // execEWMA is an exponentially-weighted moving average (alpha 1/8) of + // batch execution time in nanoseconds — the engine's own view of the + // server round-trip. It scales awaitExpectedArrivals's silence fallback so a + // wave staggered by scheduling on a slow link is not split mid-landing. Updates + // are racy read-modify-writes by design: losing an occasional sample is + // harmless for a smoothing heuristic. 0 means "no sample yet". + execEWMA atomic.Int64 + + // Lifecycle + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup // Tracks flusher goroutines + batchWg sync.WaitGroup // Tracks batch execution goroutines + // divertWg tracks the goroutines that execute DIVERTED commands (blocking + // and connection-hostile ones, which never enter a batch). Close waits on + // it exactly like batchWg so a diverted command's pooled connection is not + // left in flight after Close returns — bounded, see Close. + // + // divertMu serializes "observe not-closed, then register" against Close's + // "mark closed, then wait": without it a diverted command could pass the + // closed check, Close could see a zero counter and return, and only then + // would the goroutine register — leaving an accepted command holding a + // pooled connection past Close (and racing WaitGroup Add against Wait). + divertMu sync.Mutex + divertWg sync.WaitGroup + closed atomic.Bool +} + +// apShard is one queue + flusher. Its fields are touched only by enqueuing +// goroutines (under mu) and by its own single flusher goroutine. +// apEnqueueStripes is how many enqueue stripes a shard runs when striping is +// safe (unordered configs, and every blocking-face shard — a blocking caller +// waits for each command, so stripes cannot reorder its stream). The +// enqueue mutex is the hottest lock in the engine (128 concurrent callers on +// one shard spend ~half their CPU in lock slow paths); striping the queue +// spreads that contention while the flusher still drains every stripe into ONE +// merged pipeline, so batches stay deep. Ordered shards always use a single +// stripe: with several stripes a caller's consecutive commands can land in +// stripes on opposite sides of an in-progress drain and execute out of order. +const apEnqueueStripes = 8 + +// apStripe is one striped slice of a shard's enqueue queue. Each stripe has +// its own batch-completion signal so a drain can take stripes one lock at a +// time; every batch taken in one drain completes together after the merged +// pipeline executes. Padded so neighbouring stripes' mutexes do not share a +// cache line. +type apStripe struct { + mu sync.Mutex + queue []Cmder + queueLen atomic.Int32 + // queueBytes approximates the queued commands' payload volume; maintained + // only when MaxBatchBytes is configured (see cmdApproxBytes). + queueBytes atomic.Int64 + curBatch *apBatch // completion signal for currently-queued cmds + // Pad each stripe onto its own cache line(s). Without it, one stripe's hot + // fields (queueLen/curBatch) share a cache line with the NEXT stripe's + // contended mutex, so a lock-free counter bump on stripe i invalidates the + // line a different core is trying to lock stripe i+1 on — false sharing + // that measured ~16x on a contended microbenchmark. cpu.CacheLinePad is + // sized per GOARCH (64 B on x86-64/arm64, 128 B on ppc64, 256 B on s390x), + // so this is correct on every target rather than a hand-tuned constant. + _ cpu.CacheLinePad +} + +type apShard struct { + ap *AutoPipeliner + + next atomic.Uint32 // round-robin stripe pick (unordered mode) + stripes []apStripe // 1 stripe when ordered, apEnqueueStripes when Unordered + notify chan struct{} // buffered (cap 1) enqueue wake-up + sem *internal.FIFOSemaphore // per-shard concurrent-batch budget + + // inFlight counts this shard's dispatched-but-unfinished batches. When it + // is zero and no arrivals are expected, the shard is idle and a + // new command flushes immediately; when batches are in flight, arrivals + // are mid-stream and the flusher holds them briefly to coalesce (see + // awaitExpectedArrivals). + inFlight atomic.Int32 +} + +// stripe picks the enqueue stripe for the next command: the single stripe in +// ordered mode (preserving strict FIFO), round-robin in unordered mode. +func (s *apShard) stripe() *apStripe { + if len(s.stripes) == 1 { + return &s.stripes[0] + } + return &s.stripes[s.next.Add(1)%uint32(len(s.stripes))] +} + +// getOrCreateAutoPipeliner is the shared caching protocol behind the four +// AutoPipeline/AsyncAutoPipeline getters (Client and ClusterClient, each +// face): return the cached live instance, refuse on a closed client, or build +// and cache a new one. The caller supplies its cached-slot pointer, its +// closed flag (both guarded by the mutex), the explicit-config override, the +// fallback config, and a build closure (the cluster one wraps +// clusterAutoPipelineOptions and installs slot sharding). +func getOrCreateAutoPipeliner( + mu *sync.Mutex, + slot **AutoPipeliner, + closed *bool, + sharedClosed *atomic.Bool, + override *AutoPipelineOptions, + fallback func() *AutoPipelineOptions, + build func(*AutoPipelineOptions) (*AutoPipeliner, error), +) (*AutoPipeliner, error) { + mu.Lock() + defer mu.Unlock() + // closed covers THIS wrapper's Close; sharedClosed covers the shared + // pools closing through ANY sharer (e.g. a WithTimeout clone falling + // through to baseClient.Close) — a fresh pipeliner against closed pools + // would leak flushers that error forever. + if *closed || (sharedClosed != nil && sharedClosed.Load()) { + return nil, ErrClosed + } + if *slot != nil && !(*slot).closed.Load() { + return *slot, nil + } + cfg := override + if cfg == nil { + cfg = fallback() + } + ap, err := build(cfg) + if err != nil { + return nil, err + } + // Thread the shared pool-set closed flag into the pipeliner so an + // ALREADY-cached instance also refuses enqueues once any sharer closes + // the pools (the check above only protects fresh builds). + ap.sharedClosed = sharedClosed + *slot = ap + return ap, nil +} + +// newAutoPipeliner builds an autopipeliner in either blocking or deferred mode. +// It is unexported on purpose: the public entry points are +// Client/ClusterClient.AutoPipeline and AsyncAutoPipeline, which also install +// cluster slot-sharding. Constructing one directly would skip that wiring and +// give a *ClusterClient degraded (cross-node) batching. +func newAutoPipeliner(pipeliner cmdableClient, config *AutoPipelineOptions, blocking bool) (*AutoPipeliner, error) { + if config == nil { + config = DefaultAutoPipelineOptions() + } else { + // Copy so default-filling below doesn't mutate the caller's struct — the + // same *AutoPipelineOptions may be shared across clients (e.g. a reused + // Options.AutoPipelineOptions), and callers may inspect it afterward. + cfgCopy := *config + config = &cfgCopy + } + + // Validate BEFORE default-filling: Validate treats zero as "use the + // default" but rejects negatives, and coercing first would silently + // swallow a negative typo the documented contract promises to error on. + if err := config.Validate(); err != nil { + return nil, err + } + + // Apply defaults for zero values + if config.MaxBatchSize <= 0 { + config.MaxBatchSize = 200 + } + + if config.MaxConcurrentBatches <= 0 { + // Default to an ordered single stream. Callers raise this (with + // Unordered:true) to opt into parallel-batch throughput. + config.MaxConcurrentBatches = 1 + } + + // NumShards > 1 on the deferred (async) face distributes commands + // round-robin across shards that flush concurrently, so submit order is + // not preserved — require the explicit Unordered opt-in, exactly like + // MaxConcurrentBatches > 1. The blocking face is exempt (each caller waits + // per command, and Submit is rejected there), as is cluster slot sharding + // (contentSharded: same-key commands always land in the same shard, so + // per-key order holds). + if config.NumShards > 1 && !config.Unordered && !blocking && !config.contentSharded { + return nil, fmt.Errorf( + "redis: AutoPipelineOptions.NumShards=%d requires Unordered:true on the deferred (async) face "+ + "(commands are distributed round-robin across shards, which flush concurrently and do not preserve submit order)", + config.NumShards) + } + + ctx, cancel := context.WithCancel(context.Background()) + + ap := &AutoPipeliner{ + pipeliner: pipeliner, + config: config, + blocking: blocking, + ctx: ctx, + cancel: cancel, + } + + // Route the typed command surface. Blocking: the command call blocks until + // executed (synchronous drop-in shape). Deferred: the call returns at once + // and the result accessors block until the batch executes. + if blocking { + ap.cmdable = ap.processBlocking + } else { + ap.cmdable = ap.processAsync + } + + // Pick the shard count. NumShards=0 (auto) means ONE shard: a single deep + // queue outperforms a sharded one because batches stay large — sharding by + // core count coupled batch fragmentation to MaxConcurrentBatches and + // collapsed pipelining (measured: 16 shards cut async throughput ~4x and + // tripled latency versus one shard at the same permit count). Cluster + // wiring passes an explicit NumShards so slot-routed shards keep each + // batch on one node. + nShards := config.NumShards + if nShards <= 0 { + nShards = 1 + } + // Split the concurrent-batch budget across shards so each shard has its own + // semaphore. A single shared semaphore became a contention point once the + // per-shard queue mutexes were no longer the bottleneck. Integer division + // drops a remainder, so hand the leftover permits to the first shards: the + // per-shard permits then sum to exactly MaxConcurrentBatches. + perShard := config.MaxConcurrentBatches / nShards + remainder := config.MaxConcurrentBatches % nShards + if perShard < 1 { + // Budget smaller than the shard count: give every shard one permit so + // each flusher can still make progress. The sum then exceeds the + // configured budget, which is unavoidable with per-shard semaphores. + perShard = 1 + remainder = 0 + } + ap.shards = make([]*apShard, nShards) + for i := range ap.shards { + permits := perShard + if i < remainder { + permits++ + } + // Stripe when reordering is impossible or waived: a BLOCKING caller + // waits for each command before issuing its next, so its per-goroutine + // order holds no matter which stripe each command lands in; the async + // face may only stripe when the user set Unordered. The remaining case + // (async, ordered) keeps one stripe to preserve strict submit order. + nStripes := 1 + if config.Unordered || blocking { + nStripes = apEnqueueStripes + } + s := &apShard{ + ap: ap, + notify: make(chan struct{}, 1), + stripes: make([]apStripe, nStripes), + sem: internal.NewFIFOSemaphore(int32(permits)), + } + for j := range s.stripes { + s.stripes[j].queue = getQueueSlice(config.MaxBatchSize) + s.stripes[j].curBatch = newAPBatch() + } + ap.shards[i] = s + ap.wg.Add(1) + go s.flusher() + } + + return ap, nil +} + +// Do executes a raw command on a NORMAL connection, outside the pipeline. +// Arbitrary command names can carry connection state (SELECT, MULTI, SUBSCRIBE, +// CLIENT ...) or block the connection (BLPOP ...); batching those onto a shared +// pipeline connection would silently poison it for every later batch, or stall +// unrelated commands. (Submit enforces the same rule for raw Cmders: names in +// the connection-hostile set are diverted off the pipeline automatically.) +// The typed surface (ap.Set, ap.Get, ...) is safe by +// construction and IS batched — prefer it. Do carries the same caveats as +// Client.Do: a stateful command still affects the (normal, non-pipeline) +// pooled connection it runs on. Do keeps each face's call shape: on +// a blocking autopipeliner the call blocks until the command has executed; on a +// deferred (async) one it returns immediately and the command's result +// accessors (Err/Val/Result) block until it completes. +func (ap *AutoPipeliner) Do(ctx context.Context, args ...interface{}) *Cmd { + cmd := NewCmd(ctx, args...) + if len(args) == 0 { + cmd.SetErr(errDoNoArgs) + return cmd + } + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return cmd + } + + // Both faces go through runOutsidePipeline: it applies the divert + // registration gate, so Close cannot conclude "nothing in flight" while an + // accepted raw command — a blocking one on the blocking face runs inline on + // the caller's goroutine — is still holding a pooled connection. + _ = ap.runOutsidePipeline(ctx, cmd) + return cmd +} + +// runOutsidePipeline executes an escape-hatch command (Do, DoRaw, +// DoRawWriteTo) on a normal pooled connection, outside the batching engine, +// following the face's call shape. Blocking face: synchronous Process. +// Deferred face: returns-immediately — the command runs on a background +// goroutine and a ready batch makes its result accessors block until it +// completes. The batch completes at the innermost seam (under the user +// hooks) so a ProcessHook reading the result cannot self-deadlock; the +// deferred close is the panic backstop. Tracked by divertWg under divertMu, +// so Close waits for accepted diverted work (bounded — see Close) instead of +// returning while it still holds a pooled connection. +func (ap *AutoPipeliner) runOutsidePipeline(ctx context.Context, cmd Cmder) *apBatch { + if ap.blocking { + // The blocking face runs it inline, so the caller's own goroutine holds + // the connection; still take the gate so Close cannot decide "nothing + // in flight" while this command is executing. + ap.divertMu.Lock() + if ap.isClosed() { + ap.divertMu.Unlock() + cmd.SetErr(ErrClosed) + return completedBatch + } + ap.divertWg.Add(1) + ap.divertMu.Unlock() + defer ap.divertWg.Done() + _ = ap.pipeliner.Process(ctx, cmd) + return completedBatch + } + // Register under divertMu with a closed re-check, so registration and the + // close transition cannot interleave (see the divertMu comment). A command + // that loses the race is rejected here rather than running after Close. + // The gate comes BEFORE setReady: publishing the fresh batch first and then + // rejecting would leave the command gated on a batch nobody ever closes, + // hanging every accessor. + ap.divertMu.Lock() + if ap.isClosed() { + ap.divertMu.Unlock() + cmd.SetErr(ErrClosed) + cmd.setReady(completedBatch) + return completedBatch + } + b := newAPBatch() + cmd.setReady(b) + ap.divertWg.Add(1) + ap.divertMu.Unlock() + go func() { + defer ap.divertWg.Done() + defer b.close() + defer recoverDispatchPanic([]Cmder{cmd}) + if ap.armSelfDeadlockGuard() { + b.dispGid.Store(curGoroutineID()) + } + // A hook that returns nil WITHOUT calling next has short-circuited + // SUCCESSFULLY (it served the command itself); plain Client hooks may do + // that, so nothing here synthesizes an error for it — see dispatchCmds. + err := ap.pipeliner.withProcessHook(ctx, cmd, func(ctx context.Context, cmd Cmder) error { + return ap.pipeliner.process(ctx, cmd) + }) + // The chain's final verdict, exactly like Client.Process — recorded + // before the deferred close wakes the reader, so short-circuits, + // post-next rewrites and suppressions are all honored. + cmd.SetErr(err) + }() + return b +} + +// DoRaw mirrors Do for raw RESP access: AutoPipeliner embeds cmdable, so +// without this override DoRaw would ride the batching engine — but raw +// commands carry Do's caveats and DoRawWriteTo-style streaming must not run +// inside a shared batch's reply loop. Runs outside the pipeline, following +// the face's call shape (see Do). +func (ap *AutoPipeliner) DoRaw(ctx context.Context, args ...interface{}) *RawCmd { + cmd := NewRawCmd(ctx, args...) + if len(args) == 0 { + cmd.SetErr(errDoNoArgs) + return cmd + } + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return cmd + } + _ = ap.runOutsidePipeline(ctx, cmd) + return cmd +} + +// DoRawWriteTo mirrors Do for streamed raw RESP access (see DoRaw). On the +// deferred face the write to w happens when the command executes; use the +// result accessors (Err/Written) to wait before reading w. +func (ap *AutoPipeliner) DoRawWriteTo(ctx context.Context, w io.Writer, args ...interface{}) *RawWriteToCmd { + cmd := NewRawWriteToCmd(ctx, w, args...) + if len(args) == 0 { + cmd.SetErr(errDoNoArgs) + return cmd + } + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return cmd + } + _ = ap.runOutsidePipeline(ctx, cmd) + return cmd +} + +// Process queues a command for autopipelined execution, following the +// autopipeliner's mode like the typed methods and Do: on a blocking +// autopipeliner the call blocks until the command has executed; on a deferred +// (async) one it returns immediately and reading the command's result +// (Val/Result/Err) blocks until its batch is flushed. +func (ap *AutoPipeliner) Process(ctx context.Context, cmd Cmder) error { + return ap.cmdable(ctx, cmd) +} + +// The methods below complete the UniversalClient surface by delegating to the +// underlying client. They are NOT autopipelined — pub/sub, transactions (Watch), +// hooks, Do and pool stats cannot be batched — so an AutoPipeliner used as a +// UniversalClient batches only the typed data commands; everything here runs on +// the underlying client exactly as it would there. +// +// Note on lifecycle: Close() (defined elsewhere) closes the AUTOPIPELINER — +// drains in-flight batches and stops flushers — but does NOT close the +// underlying client, whose lifecycle is owned by whoever created it. + +// AddHook adds a hook to the underlying client. Autopipelined batches are hooked +// too, since dispatch goes through the hook-wrapped pipeline entry. +func (ap *AutoPipeliner) AddHook(hook Hook) { ap.pipeliner.AddHook(hook) } + +// The four commands below have CLUSTER-WIDE overrides on ClusterClient +// (DBSize sums every master, the Script commands fan out to every shard). +// The embedded generic cmdable would route them as ordinary keyless commands +// to one picked shard — partial results, scripts missing on other shards — +// so they delegate to the underlying client instead of batching. On a +// standalone client the delegation is semantically identical to the generic +// path; these are rare admin/script-management commands, not data-path. + +// DBSize delegates to the underlying client (cluster-wide sum on ClusterClient). +func (ap *AutoPipeliner) DBSize(ctx context.Context) *IntCmd { + return ap.pipeliner.DBSize(ctx) +} + +// ScriptLoad delegates to the underlying client (loads every shard on ClusterClient). +func (ap *AutoPipeliner) ScriptLoad(ctx context.Context, script string) *StringCmd { + return ap.pipeliner.ScriptLoad(ctx, script) +} + +// ScriptFlush delegates to the underlying client (flushes every shard on ClusterClient). +func (ap *AutoPipeliner) ScriptFlush(ctx context.Context) *StatusCmd { + return ap.pipeliner.ScriptFlush(ctx) +} + +// ScriptExists delegates to the underlying client (ANDs results across shards +// on ClusterClient). +func (ap *AutoPipeliner) ScriptExists(ctx context.Context, hashes ...string) *BoolSliceCmd { + return ap.pipeliner.ScriptExists(ctx, hashes...) +} + +// HImportPrepare, HImportDiscard and HImportDiscardAll are the remaining +// cluster-wide overrides (see the delegation note above): ClusterClient fans +// them out to every master and updates the shared fieldset registry, so +// running them on a single routed node would let a later HImportSet for a key +// on another master fail with "no such fieldset". TestAPDelegatesClusterWideOverrides +// fails if a future ClusterClient override is added without a delegate here. +func (ap *AutoPipeliner) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd { + return ap.pipeliner.HImportPrepare(ctx, fieldsetName, fields...) +} + +func (ap *AutoPipeliner) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd { + return ap.pipeliner.HImportDiscard(ctx, fieldsetName) +} + +func (ap *AutoPipeliner) HImportDiscardAll(ctx context.Context) *IntCmd { + return ap.pipeliner.HImportDiscardAll(ctx) +} + +// Watch runs a transactional function on the underlying client (not batched). +func (ap *AutoPipeliner) Watch(ctx context.Context, fn func(*Tx) error, keys ...string) error { + return ap.pipeliner.Watch(ctx, fn, keys...) +} + +// Subscribe opens a pub/sub on the underlying client (not batched — pub/sub +// needs a dedicated connection). +func (ap *AutoPipeliner) Subscribe(ctx context.Context, channels ...string) *PubSub { + return ap.pipeliner.Subscribe(ctx, channels...) +} + +// PSubscribe opens a pattern pub/sub on the underlying client (not batched). +func (ap *AutoPipeliner) PSubscribe(ctx context.Context, channels ...string) *PubSub { + return ap.pipeliner.PSubscribe(ctx, channels...) +} + +// SSubscribe opens a sharded pub/sub on the underlying client (not batched). +func (ap *AutoPipeliner) SSubscribe(ctx context.Context, channels ...string) *PubSub { + return ap.pipeliner.SSubscribe(ctx, channels...) +} + +// PoolStats returns the underlying client's connection pool statistics. +func (ap *AutoPipeliner) PoolStats() *PoolStats { return ap.pipeliner.PoolStats() } + +// AutoPipeline delegates to the underlying client, which returns its cached +// autopipeliner (typically this same instance). Present to satisfy the +// UniversalClient surface. +func (ap *AutoPipeliner) AutoPipeline() (*AutoPipeliner, error) { + return ap.pipeliner.AutoPipeline() +} + +// AutoPipelineWithOptions delegates to the underlying client. +func (ap *AutoPipeliner) AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return ap.pipeliner.AutoPipelineWithOptions(config) +} + +// AsyncAutoPipeline delegates to the underlying client. Present to satisfy the +// UniversalClient surface. +func (ap *AutoPipeliner) AsyncAutoPipeline() (*AutoPipeliner, error) { + return ap.pipeliner.AsyncAutoPipeline() +} + +// AsyncAutoPipelineWithOptions delegates to the underlying client. +func (ap *AutoPipeliner) AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return ap.pipeliner.AsyncAutoPipelineWithOptions(config) +} + +// AutoFuture is the handle returned by Submit. Call Wait (or Result on the +// command after Wait) once the result is needed; it blocks only until the +// command's batch has executed. +type AutoFuture struct { + cmd Cmder + batch *apBatch +} + +// Wait blocks until the submitted command has executed, then returns its error. +// The zero AutoFuture (no submitted command) returns an error rather than +// panicking. +func (f AutoFuture) Wait() error { + if f.batch == nil { + if f.cmd != nil { + return f.cmd.Err() + } + return errZeroAutoFuture + } + select { + case <-f.batch.done: + default: + // Same self-deadlock guard as baseCmd.await(): a pipeline hook on + // the batch's own dispatch goroutine waiting a future pre-next() + // would block a channel only its goroutine can close. Give it the + // not-yet-executed view instead. + if f.batch.isExecutorGoroutine() { + return f.cmd.rawErr() + } + <-f.batch.done + } + return f.cmd.Err() +} + +// WaitContext is like Wait but stops waiting when ctx is done. The command +// still executes and its result remains readable once its batch completes — +// ctx abandons only this wait, it does not cancel the command (per-command +// contexts are not honored after enqueue; see the AutoPipeliner doc). +// +// After a ctx error the result may simply not be there YET: the batch is +// still in flight and may populate the command at any moment, so do not read +// Cmd()'s value or error directly — that races the executing batch. Call Wait +// (or WaitContext with a fresh context) again; once it returns a non-context +// error, the command's result is complete and safe to read. +func (f AutoFuture) WaitContext(ctx context.Context) error { + if f.batch == nil { + if f.cmd != nil { + return f.cmd.Err() + } + return errZeroAutoFuture + } + select { + case <-f.batch.done: + return f.cmd.Err() + default: + if f.batch.isExecutorGoroutine() { + return f.cmd.rawErr() // see Wait: dispatch-goroutine self-deadlock guard + } + } + select { + case <-f.batch.done: + return f.cmd.Err() + case <-ctx.Done(): + return ctx.Err() + } +} + +// Cmd returns the underlying command (call Wait first before reading results). +func (f AutoFuture) Cmd() Cmder { return f.cmd } + +// outsidePipelineCommands lists commands that must never ride a SHARED +// pipeline connection. SHUTDOWN terminates the server before replying (its +// batchmates would all fail with EOF and the batch would retry against a +// dead server); MONITOR rebinds the connection into a monitor stream, +// desyncing every reply behind it; the rest change per-connection state +// (database, auth, protocol, transaction, subscription mode) that would +// leak to every unrelated caller sharing the pipeline conn afterwards. The +// typed surface cannot produce most of the stateful ones (they live on +// statefulCmdable) — but ReadOnly/ReadWrite ARE on cmdable, and raw +// Submit/Do accept any Cmder. Diverted commands execute directly on their +// own pooled connection — the same semantics (including the same footguns) +// as plain Client.Do. +var outsidePipelineCommands = map[string]struct{}{ + "shutdown": {}, "monitor": {}, + "select": {}, "auth": {}, "hello": {}, "reset": {}, "quit": {}, + "multi": {}, "exec": {}, "discard": {}, "watch": {}, "unwatch": {}, + "subscribe": {}, "unsubscribe": {}, "psubscribe": {}, "punsubscribe": {}, + "ssubscribe": {}, "sunsubscribe": {}, + "client": {}, + // Connection-scoped cluster state: queued onto a shared pipeline conn + // they would leak replica-reads (or a pending redirect) to every later + // batch on that conn. + "readonly": {}, "readwrite": {}, "asking": {}, +} + +func runsOutsidePipeline(name string) bool { + _, ok := outsidePipelineCommands[name] + return ok +} + +// blockingCommands are commands that park on the server until data arrives or +// their own timeout expires. The TYPED helpers set a per-command read timeout +// (see cmdable.BLPop), which submit already diverts on; a RAW Cmder built by +// hand — NewCmd(ctx, "blpop", key, 0) via Submit/Process/Do — carries no such +// marker, so without this set it would be queued onto a shared pipeline +// connection and hold the whole batch for the block duration. +// Derived from the typed helpers rather than guessed: every cmdable method that +// calls cmd.setReadTimeout parks the connection, so +// +// grep -rn 'setReadTimeout' --include='*.go' . | grep -v _test +// +// enumerates exactly the wire names that belong here (the arg-driven ones are +// handled in isBlockingCmd instead). Re-run that grep when adding a blocking +// command. +var blockingCommands = map[string]struct{}{ + "blpop": {}, "brpop": {}, "brpoplpush": {}, + "blmove": {}, "blmovem": {}, "blmpop": {}, + "bzpopmin": {}, "bzpopmax": {}, "bzmpop": {}, + "wait": {}, "waitaof": {}, + // MIGRATE blocks the source instance for up to its timeout. + "migrate": {}, +} + +// isBlockingCmd reports whether cmd parks the connection. XREAD/XREADGROUP are +// decided by ARGUMENTS, not by name: only the BLOCK form blocks, and +// blanket-diverting the (far more common) non-blocking form would drop it out +// of batching for nothing. +func isBlockingCmd(cmd Cmder) bool { + name := cmd.Name() + if _, ok := blockingCommands[name]; ok { + return true + } + // Arg-driven: these block only in their BLOCK form, and blanket-diverting + // the far more common non-blocking form would drop it out of batching for + // nothing. TS.READ takes BLOCK the same way (see TSReadWithArgs). + if name != "xread" && name != "xreadgroup" && name != "ts.read" { + return false + } + // Match the token the way the encoder does: a raw Cmder may carry RESP + // tokens as []byte or *string (see baseCmd.stringArg), and a type switch on + // string alone would let NewCmd(ctx, "xread", []byte("BLOCK"), 0, ...) be + // batched onto a shared connection. + for _, arg := range cmd.Args() { + if internal.ToLower(blockingArgString(arg)) == "block" { + return true + } + } + return false +} + +// blockingArgString renders a command argument as the string the encoder will +// write for the token comparisons above. Only the forms that can carry a RESP +// keyword are handled; anything else cannot be the BLOCK token. +func blockingArgString(arg interface{}) string { + switch v := arg.(type) { + case string: + return v + case []byte: + return string(v) + case *string: + if v == nil { + return "" + } + return *v + default: + return "" + } +} + +// submit queues a command without blocking and returns its completion future. +func (ap *AutoPipeliner) submit(ctx context.Context, cmd Cmder) AutoFuture { + // finish marks the command ready on the deferred face so its result + // accessors (Val/Result/Err) self-gate through await() — whether the + // caller goes through the typed surface or raw Submit. Reading a + // Submit()-ed command before Wait() was previously a silent data race + // with the dispatch goroutine. The blocking face deliberately never + // carries a batch: its callers only regain control after execution, and + // the dispatcher-gid deadlock guard relies on that. + finish := func(f AutoFuture) AutoFuture { + if !ap.blocking { + cmd.setReady(f.batch) + } + return f + } + // Decide DIVERSION first. The cluster preflight rejects commands whose + // request policy cannot ride a pipeline (ReqAllNodes/ReqAllShards), but a + // diverted command never rides one: it goes through the underlying + // Client/ClusterClient.Process, which performs the normal cluster-wide + // fan-out and aggregation. Running the preflight first therefore rejected + // commands that would have worked — typed WAIT/WAITAOF on a cluster with + // command policies enabled (review finding by codex on #3942). + diverted := cmd.readTimeout() != nil || runsOutsidePipeline(cmd.Name()) || isBlockingCmd(cmd) || + (ap.mustDivert != nil && ap.mustDivert(ctx, cmd)) + if !diverted && ap.preflight != nil { + if err := ap.preflight(ctx, cmd); err != nil { + cmd.SetErr(err) + return finish(AutoFuture{cmd: cmd, batch: completedBatch}) + } + } + if diverted { + // Blocking commands (and the conn-hostile ones above) are executed + // directly, outside the pipeline — via runOutsidePipeline, which + // keeps each face's call shape: the blocking face runs the command + // synchronously, the deferred face runs it on its own goroutine so + // this call returns immediately and the result accessors block (a + // BLPOP submitted on the async face must not stall the submitter, + // exactly like Do). They still must respect a closed AutoPipeliner: + // enqueue() rejects on the batched path, so mirror that here instead + // of running after Close(). + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return finish(AutoFuture{cmd: cmd, batch: completedBatch}) + } + // runOutsidePipeline sets the command ready itself on the deferred + // face; the returned batch completes when the command has executed. + return AutoFuture{cmd: cmd, batch: ap.runOutsidePipeline(ctx, cmd)} + } + // No finish here: enqueue stamps ready under the stripe lock, before the + // command is visible to any drain (the error paths above still go through + // finish for uniform accessor behavior). + return AutoFuture{cmd: cmd, batch: ap.enqueue(cmd)} +} + +// ErrSubmitBlockingFace rejects Submit on the blocking face: Submit does not +// wait, so a windowed caller could have several commands in flight at once — +// but the blocking face stripes its enqueue queue on the strength of every +// caller waiting per command, and a non-waiting window there can be reordered. +// The deferred face (AsyncAutoPipeline) is built for exactly that usage. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +var ErrSubmitBlockingFace = errors.New( + "redis: Submit requires the deferred autopipeliner (AsyncAutoPipeline); on the blocking face use the typed methods or Do") + +// errZeroAutoFuture is returned by Wait/WaitContext on a zero AutoFuture. +var errZeroAutoFuture = errors.New("redis: Wait on a zero AutoFuture") + +// errDoNoArgs is returned by Do when called without a command. +var errDoNoArgs = errors.New("redis: AutoPipeliner.Do requires at least one argument") + +// ErrAutoPipelineTimeout is set on drained commands when a flush could not +// obtain a batch permit within the engine's internal backstop — the engine is +// overloaded or an in-flight batch is wedged (e.g. read timeouts disabled on +// a dead peer). It is deliberately NOT context.DeadlineExceeded: the caller's +// own context did not expire, and errors.Is(err, context.DeadlineExceeded) +// must not fire for an internal engine timeout. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +var ErrAutoPipelineTimeout = errors.New( + "redis: autopipeline: no batch permit within the internal backstop (engine overloaded or a batch is wedged)") + +// Submit queues a command without blocking and returns an AutoFuture; Wait on +// it when the result is needed. This is the explicit form for working with raw +// Cmders on the deferred (async) face, where the typed methods (Set, Get, ...) +// provide the same deferred behaviour returning the usual *XxxCmd. The +// command's own result accessors (Err/Val/Result) are safe to use instead of +// Wait — they block until the command has executed. Connection-hostile +// command names (SHUTDOWN, MONITOR, SELECT, AUTH, MULTI, SUBSCRIBE, CLIENT, +// ...) never ride a shared pipeline connection: they are diverted to a +// normal pooled connection with plain Client.Do semantics. On a BLOCKING +// autopipeliner Submit is rejected (the future's Wait returns an error): the +// blocking face's ordering relies on every caller waiting for each command +// before issuing the next, which Submit by design does not do. +func (ap *AutoPipeliner) Submit(ctx context.Context, cmd Cmder) AutoFuture { + if ap.blocking { + cmd.SetErr(ErrSubmitBlockingFace) + return AutoFuture{cmd: cmd, batch: completedBatch} + } + return ap.submit(ctx, cmd) +} + +// processAsync is the cmdable backing the typed command surface: it queues a +// command without blocking the caller and marks it ready so the command's +// result accessors (Val/Result/Err) block until the batch executes. This gives +// the autopipeliner the full typed surface (ap.Set, ap.Get, ...) with the exact +// same call shape as a normal client — only the wait is deferred to the point a +// result is read. +func (ap *AutoPipeliner) processAsync(ctx context.Context, cmd Cmder) error { + // submit marks the command ready (see the finish closure there): a hook + // that reads the command before that store lands sees a nil ready — the + // non-blocking not-yet-executed view — while the caller always sees its + // own store before any await. + f := ap.submit(ctx, cmd) + // Report SUBMIT-time rejections (a closed pipeliner, a cluster preflight + // refusal): those paths set the error on the command and hand back the + // shared completed batch without queueing anything, so returning nil made + // Process claim success for a command that will never run — and callers + // reaching the engine through UniversalClient.Process see only this return + // value (review finding by codex on #3942). Execution errors are NOT + // reported here: the deferred face's contract is that this call does not + // wait, so those stay on the command for its accessors. rawErr keeps the + // check non-blocking. + if f.batch == completedBatch { + return cmd.rawErr() + } + return nil +} + +// processBlocking is the cmdable backing the blocking face: it queues the +// command and blocks until its batch has executed, so the command call has the +// same synchronous shape as a normal client (the returned *XxxCmd already holds +// its result). The flusher still batches this command with other concurrent +// callers' commands into a pipeline, so throughput is far above a plain client +// even though each caller waits. Per-goroutine ordering holds regardless of +// MaxConcurrentBatches: a caller cannot issue its next command until this one +// returns, so its commands execute in submit order. +func (ap *AutoPipeliner) processBlocking(ctx context.Context, cmd Cmder) error { + return ap.submit(ctx, cmd).Wait() +} + +// completedBatch is a reusable already-completed batch: returned both for +// commands that already executed directly (blocking commands, Submit-time +// rejections) and for error cases like enqueue-after-Close, so Wait returns +// immediately and the command's own error tells the story. +var completedBatch = func() *apBatch { + b := newAPBatch() + b.close() + return b +}() + +// enqueue queues a command and returns the batch whose done channel completes +// when it has executed. On a closed autopipeliner it errors the command and +// returns the already-closed batch. +// isClosed reports whether this pipeliner (or the shared pool set it rides +// on) has been closed. Two atomic loads; no locks. +// +// EVERY closed check that gates accepting new work must go through this, not +// ap.closed directly: a WithTimeout clone's Close sets only the shared flag, +// so a guard reading ap.closed alone would accept commands against pools that +// are already gone and surface pool-closed errors instead of ErrClosed. +// (Close's own CompareAndSwap on ap.closed is the one deliberate direct use: +// it claims the shutdown for this instance.) +func (ap *AutoPipeliner) isClosed() bool { + return ap.closed.Load() || (ap.sharedClosed != nil && ap.sharedClosed.Load()) +} + +func (ap *AutoPipeliner) enqueue(cmd Cmder) *apBatch { + if ap.isClosed() { + cmd.SetErr(ErrClosed) + return completedBatch + } + + // Pick a shard. With shardFn (cluster mode) route by command content so all + // commands for one node collect in the same shard's batch; otherwise spread + // round-robin to keep each shard's mutex lightly contended. + var s *apShard + if ap.shardFn != nil { + // uint conversion instead of negation: -math.MinInt overflows back to + // itself and a negative modulo would panic the index. The unsigned + // modulo is deterministic for every int, including MinInt. + idx := ap.shardFn(cmd) + s = ap.shards[uint(idx)%uint(len(ap.shards))] + } else if len(ap.shards) == 1 { + // Single shard (the standalone default): skip the round-robin counter — + // it is a shared cache line bumped by every enqueue for a pick that is + // constant. Same guard the stripe pick already has. + s = ap.shards[0] + } else { + // Unsigned modulo: converting to int first goes negative after the + // uint32 counter passes 2^31 on 32-bit platforms and panics. + s = ap.shards[int((ap.next.Add(1)-1)%uint32(len(ap.shards)))] + } + + st := s.stripe() + st.mu.Lock() + // Re-check closed under the stripe lock (see Close): either we win the lock + // first and the shutdown drain flushes us, or the drain ran first and we + // reject here — so a late enqueue never hangs on an unclosed done. + if ap.isClosed() { + st.mu.Unlock() + cmd.SetErr(ErrClosed) + return completedBatch + } + batch := st.curBatch + if !ap.blocking { + // Publish the gating batch BEFORE the command becomes visible to a + // drain (the drain takes this same stripe lock): a flush racing the + // submitter's return path must observe ready already set, or the + // cluster node-executor registration would skip this command's batch + // and a node hook reading the command mid-dispatch could block on a + // batch its own call chain completes. The blocking face deliberately + // never carries a batch (see submit). + cmd.setReady(batch) + } + st.queue = append(st.queue, cmd) + st.queueLen.Store(int32(len(st.queue))) + if ap.config.MaxBatchBytes > 0 { + st.queueBytes.Add(cmdApproxBytes(cmd)) + } + st.mu.Unlock() + + // One expected arrival has landed (see expectedArrivals). + ap.expectedArrivals.Add(-1) + + s.wake() + return batch +} + +// wake signals the shard's flusher that work is available without blocking. +func (s *apShard) wake() { + select { + case s.notify <- struct{}{}: + default: + } +} + +// IsBlocking reports which face this autopipeliner is: true for the blocking +// face (Client.AutoPipeline — calls wait for execution), false for the +// deferred face (AsyncAutoPipeline — calls return immediately and result +// accessors block). The two faces reject different usage (Submit is +// blocking-face-rejected), so code handed an *AutoPipeliner can branch on +// this instead of probing with errors. +func (ap *AutoPipeliner) IsBlocking() bool { return ap.blocking } + +// Config returns a copy of the effective configuration (defaults filled in). +func (ap *AutoPipeliner) Config() AutoPipelineOptions { + cfg := *ap.config + // Strip internal-only fields. contentSharded is set by cluster wiring and + // tells Validate that shards are slot-routed, so same-key commands cannot + // be reordered — which exempts the config from the NumShards>1 ordering + // requirement. Handing that bit back to a caller who copies this config + // into a STANDALONE async autopipeliner would silence that check for + // round-robin shards, which really do flush concurrently and really do + // break submit order (review finding by codex on #3942). + cfg.contentSharded = false + return cfg +} + +// IsClosed reports whether the AutoPipeliner has been closed, either by an +// explicit Close or by closing the owning client. A closed AutoPipeliner +// rejects new commands with ErrClosed. +func (ap *AutoPipeliner) IsClosed() bool { + return ap.isClosed() +} + +// numShards reports how many shards this autopipeliner runs. +func (ap *AutoPipeliner) numShards() int { return len(ap.shards) } + +// setShardFn installs a content-based shard selector. In cluster mode it maps +// a command's SLOT to a shard, which is a batch-depth heuristic, not an +// invariant: slot ranges are assigned to shards proportionally, so when a +// node's slots are non-contiguous one shard's batch can still span nodes and +// mapCmdsByNode splits it (correctness is unaffected — that router resolves +// every command's own slot — but those per-node pipelines are shallower). +// What the mapping DOES guarantee is that a given key always lands on the same +// shard, so a caller's relative order for that key is preserved regardless of +// how the shard's batch is split. Must be called before the autopipeliner is +// used. Not safe to change concurrently with enqueues. +func (ap *AutoPipeliner) setShardFn(fn func(Cmder) int) { ap.shardFn = fn } + +// setPreflight installs a submit-time command filter (cluster wiring rejects +// commands whose request policy cannot ride a pipeline). Called once during +// construction, before the AutoPipeliner is published. +func (ap *AutoPipeliner) setPreflight(fn func(ctx context.Context, cmd Cmder) error) { + ap.preflight = fn +} + +// setMustDivert installs a predicate that forces a command off the batching +// path (see the mustDivert field). Called once during construction, before the +// AutoPipeliner is published. +func (ap *AutoPipeliner) setMustDivert(fn func(ctx context.Context, cmd Cmder) bool) { + ap.mustDivert = fn +} + +// Close stops the autopipeliner and flushes any pending commands. Worst +// case it blocks up to the internal permit backstop (~30s) PER SHARD if +// in-flight batches are wedged (e.g. read timeouts disabled against a dead +// peer) — healthy shutdowns take one round trip per shard with commands +// queued, near-zero otherwise. +func (ap *AutoPipeliner) Close() error { + if !ap.closed.CompareAndSwap(false, true) { + return nil // Already closed + } + + // Cancel context to stop flushers + ap.cancel() + + // Wake every shard's flusher so each observes the cancelled context promptly. + for _, s := range ap.shards { + s.wake() + } + + // Pass through the divert gate once: after the CompareAndSwap above, any + // registration either completed before this (so the counter already sees + // it) or will observe closed==true and reject. Without this handshake the + // wait below could read a zero counter while a diverted command was + // between its closed check and its Add. + ap.divertMu.Lock() + ap.divertMu.Unlock() //nolint:staticcheck // handshake, not a critical section + + // Drain everything that remains, BOUNDED AS ONE UNIT: the flusher exit, the + // final shard sweep, and the batch/diverted dispatch waits. + // + // None of it can be cancelled: commands taken from a queue (or accepted for + // diverted execution) were already ACCEPTED, and Close's contract is to + // flush them, so ap.cancel() deliberately does not reach an in-flight + // dispatch. With ReadTimeout disabled — a supported configuration — a + // stalled read against a dead peer, or a diverted BLPOP with a zero + // timeout, has nothing to end it. Bounding only the LAST wait would not + // help: the wedged dispatch can just as easily sit in a flusher that + // ap.wg.Wait() is waiting for, or in the shutdown sweep's own dispatch, so + // Close would hang before ever reaching the bound it documents (review + // finding by codex on #3942). On expiry, report what is still outstanding + // instead of blocking the caller: the engine is already closed to new work, + // and the leaked goroutines end when the server or the OS breaks the + // connection. See autoPipelineCloseBackstop for why the bound is generous. + return ap.drainAll(autoPipelineCloseBackstop) +} + +// drainAll runs Close's whole drain tail under a single bound and returns an +// error naming every stage that was still outstanding when it expired. Split +// out of Close so the bound is testable without a real stalled connection. +// +// The stages are ordered as Close needs them — the shard sweep must not start +// before the flushers are provably gone — but they are waited on +// CONCURRENTLY with the timer, which is the whole point: any stage can be the +// one that never finishes. +func (ap *AutoPipeliner) drainAll(timeout time.Duration) error { + flushers := make(chan struct{}) + go func() { defer close(flushers); ap.wg.Wait() }() + + // swept: after the flushers are gone, drain each shard once more under its + // lock. A command can pass enqueue's under-lock closed-recheck just before + // Close's CompareAndSwap and append to a shard AFTER that shard's flusher + // has already drained and exited — leaving its batch.done unclosed and the + // caller's accessor blocked forever. s.mu serializes the two, so either the + // late enqueue appends first and this sweep flushes it, or the sweep runs + // first and the enqueue then observes closed==true and rejects. + swept := make(chan struct{}) + go func() { + defer close(swept) + <-flushers + for _, s := range ap.shards { + s.flushBatchSliceShutdown() + } + }() + + batches := make(chan struct{}) + go func() { + defer close(batches) + <-swept + ap.batchWg.Wait() + }() + + diverted := make(chan struct{}) + go func() { defer close(diverted); ap.divertWg.Wait() }() + + timer := time.NewTimer(timeout) + defer timer.Stop() + batchesDone, divertedDone := false, false + for !batchesDone || !divertedDone { + select { + case <-batches: + batchesDone = true + batches = nil // a closed channel is always ready; stop selecting it + case <-diverted: + divertedDone = true + diverted = nil + case <-timer.C: + var outstanding []string + if !batchesDone { + // Name the precise stage: a wedged flusher and a wedged batch + // dispatch need different operator responses. + select { + case <-flushers: + select { + case <-swept: + outstanding = append(outstanding, "batch dispatches") + default: + outstanding = append(outstanding, "the shutdown flush") + } + default: + outstanding = append(outstanding, "the flusher drain") + } + } + if !divertedDone { + outstanding = append(outstanding, "diverted (blocking) commands") + } + return fmt.Errorf( + "redis: autopipeline: Close timed out after %s with %s still in flight; "+ + "they hold pooled connections until the server or the OS ends them "+ + "(most often a blocking command with no timeout, or ReadTimeout disabled)", + timeout, strings.Join(outstanding, " and ")) + } + } + return nil +} + +// flusher is the per-shard background goroutine that flushes batches. +func (s *apShard) flusher() { + defer s.ap.wg.Done() + ap := s.ap + + for { + // Wait for a command to arrive (or shutdown). The notify channel is a + // cheap buffered wake-up; no lock is taken on the hot enqueue path. + if s.Len() == 0 { + select { + case <-s.notify: + case <-ap.ctx.Done(): + } + } + + // Check if context is cancelled + if ap.ctx.Err() != nil { + // Final flush before shutdown - use background context to avoid immediate cancellation + s.flushBatchSliceShutdown() + return + } + + // Apply the coalescing window if one is configured (MaxFlushDelay / + // AdaptiveDelay). With the default config this returns at once: batching + // under concurrent load comes from in-flight backpressure, not a wait — + // see accumulateBatch. + s.accumulateBatch() + + // Flush all pending commands + for s.Len() > 0 { + select { + case <-ap.ctx.Done(): + // Final flush before shutdown + s.flushBatchSliceShutdown() + return + default: + } + + s.flushBatchSlice() + + // Between batches, apply the configured window again so the next + // pipeline is also full. A no-op with the default config (see + // accumulateBatch); the next drain picks up whatever has queued. + if s.Len() > 0 && s.Len() < ap.config.MaxBatchSize && !s.bytesFull() { + s.accumulateBatch() + } + } + } +} + +// accumulateBatch lets commands pile up before the flusher drains the queue, +// so pipelines carry many commands instead of one. It returns as soon as any +// of these holds: +// +// - the queue reaches MaxBatchSize (batch is full); +// - a configured MaxFlushDelay / AdaptiveDelay window elapses; or +// - with no configured window (the default), the expected resubmission +// wave of arrivals has landed — see awaitExpectedArrivals. +// +// A configured MaxFlushDelay / AdaptiveDelay is an intentional accumulation +// window and is waited in full (AdaptiveDelay scales it down as the queue fills +// and returns 0 — flush now — once the queue is ≥75% full). +func (s *apShard) accumulateBatch() { + ap := s.ap + batchSize := ap.config.MaxBatchSize + if batchSize <= 0 { + batchSize = 1 + } + if s.Len() >= batchSize || s.bytesFull() { + return + } + + // Pick the accumulation window. calculateDelay returns 0 both when no + // MaxFlushDelay is configured (the default) and when AdaptiveDelay resolves + // the current fill level to "flush immediately". The fill level is this + // shard's own length — each shard flushes independently, so a global count + // would mis-tune a quiet shard while another is busy. + window := ap.calculateDelay(s.Len()) + if window <= 0 { + if ap.config.MaxFlushDelay == 0 && !ap.config.AdaptiveDelay { + // Default: coalesce by expected-arrival count, not by wall-clock. + s.awaitExpectedArrivals(batchSize) + } + return + } + + // Explicit window: wait the whole delay (or until the batch fills). Each + // enqueue sends on notify, so we re-check the queue length on every wake-up + // and return once the batch is full. + deadline := time.NewTimer(window) + defer deadline.Stop() + for { + select { + case <-ap.ctx.Done(): + return + case <-deadline.C: + return + case <-s.notify: + if s.Len() >= batchSize || s.bytesFull() { + return + } + } + } +} + +// silenceGapFloor / silenceGapCeil bound awaitExpectedArrivals's silence fallback. +// The floor covers fast links; the RTT-scaled value (execEWMA/8) takes over on +// slow ones, where a wakeup wave staggered by goroutine scheduling can pause +// longer than the floor mid-landing and a premature flush is expensive (each +// batch fragment occupies a pipeline connection for a full round trip). The +// ceiling bounds how long a stale expectation (callers that left) can delay a +// flush. +const ( + silenceGapFloor = 200 * time.Microsecond + silenceGapCeil = 2 * time.Millisecond +) + +// coalesceMinFlush is the smallest pipeline worth dispatching while other +// batches are still executing. Below it, a gap-fire holds the queued +// stragglers for the next wave instead of burning a connection on a +// near-empty flush; once nothing is in flight, any size flushes immediately. +const coalesceMinFlush = 8 + +// observeBatchExec folds one batch execution duration into execEWMA. +func (ap *AutoPipeliner) observeBatchExec(d time.Duration) { + sample := int64(d) + if sample <= 0 { + return + } + old := ap.execEWMA.Load() + if old == 0 { + ap.execEWMA.Store(sample) + return + } + ap.execEWMA.Store(old + (sample-old)/8) +} + +// silenceGap returns the silence fallback for awaitExpectedArrivals, scaled to the +// observed batch round-trip: clamp(execEWMA/8, floor, ceil). +func (ap *AutoPipeliner) silenceGap() time.Duration { + g := time.Duration(ap.execEWMA.Load() / 8) + if g < silenceGapFloor { + return silenceGapFloor + } + if g > silenceGapCeil { + return silenceGapCeil + } + return g +} + +// awaitExpectedArrivals holds the flusher while related work is in motion, so +// commands flush as deep pipelines instead of fragmenting into small batches +// (each fragment costs a pipeline connection for a full round trip). Two +// signals — both facts the engine already has, not wall-clock guesses — decide +// whether anything is imminent: +// +// - expectedArrivals: a completed batch of N commands wakes its N waiters +// together, and in a closed loop each immediately submits its next +// command. Completion announces the exact count; every enqueue accounts +// for one; the wait ends the moment the count drains — the wave of +// arrivals has fully landed. An exact per-wave count has no failure mode +// where an averaged estimate undershoots the true wave and locks the +// engine into fragmented flushes. +// - inFlight: batches still executing mean their waiters will wake shortly +// and stragglers are mid-stream — worth holding a moment to coalesce with, +// bounded by the silence gap. This also recovers a fragmented state (many +// singles in flight, which announce nothing): their staggered returns land +// within one gap, merge into a real batch, and arrival tracking resumes. +// +// When neither holds, the shard is idle and the flush happens immediately: a +// lone caller pays a single round trip with no timer armed. That is the point +// of the design — the previous fixed ~20µs debounce timer armed on every flush +// fires ~1ms late on an idle or low-core host (wakeup latency dominates the +// requested delay), taxing every low-concurrency command ~5x its round trip. +// Here the gap timer never fires in steady state, closed loop or open; it only +// ends waits for callers that left. +func (s *apShard) awaitExpectedArrivals(batchSize int) { + ap := s.ap + expected := ap.expectedArrivals.Load() + if expected < 0 { + // Arrivals outran what was announced (open-loop traffic); re-zero so + // the deficit does not mask the next wave. CAS: only clear the value + // we saw, never a concurrent announcement. + ap.expectedArrivals.CompareAndSwap(expected, 0) + expected = 0 + } + expectingWave := expected > 0 + if !expectingWave && s.inFlight.Load() == 0 { + // Idle shard: nothing imminent, flush in one round trip. + return + } + + gap := ap.silenceGap() + // Reset is drain-safe on Go 1.23+ (see go.mod: go 1.24). + fallback := time.NewTimer(gap) + defer fallback.Stop() + lastSeenExpected := expected // count as of the most recent timer (re)arm + var holdStart time.Time // set on the first straggler-hold gap fire + for { + select { + case <-ap.ctx.Done(): + return + case <-fallback.C: + if !expectingWave && s.Len() < coalesceMinFlush && s.inFlight.Load() > 0 { + // Only stragglers queued while batches are still executing: + // flushing a near-empty pipeline burns a connection for a full + // round trip (measured at high WAN concurrency: straggler + // flushes of 1-3 commands starved the connection pool and + // doubled p50). Hold them — the next completed batch's wave + // sweeps them along, and the wave path below flushes promptly. + // The hold is bounded like the permit wait: with read timeouts + // disabled a wedged batch could pin inFlight forever, and the + // held stragglers must not hang with it. + if holdStart.IsZero() { + holdStart = time.Now() + } + if time.Since(holdStart) < autoPipelinePermitBackstop { + lastSeenExpected = ap.expectedArrivals.Load() + fallback.Reset(gap) + continue + } + } + if expectingWave { + // A whole gap passed with no arrivals on this shard: the + // expected callers left (workload shrank), so clear the stale + // expectation or future flushes will wait for ghosts. But only + // if it did not GROW during the silent gap — growth means a + // batch elsewhere (another shard, or racing this fire) + // announced a fresh wave, and erasing that would fragment a + // wave that is really coming. CAS, never a blind store, so an + // announcement racing the reset itself also survives. + if d := ap.expectedArrivals.Load(); d > 0 && d <= lastSeenExpected { + ap.expectedArrivals.CompareAndSwap(d, 0) + } + } + return + case <-s.notify: + if s.Len() >= batchSize || s.bytesFull() { + return + } + if d := ap.expectedArrivals.Load(); d > 0 { + // An in-flight batch completed mid-wait: its wave is now the + // thing to wait out, with the exact-count exit below. + expectingWave = true + lastSeenExpected = d + } else if expectingWave { + // The wave has fully landed; flush it as one batch. + return + } else if s.inFlight.Load() == 0 { + // Nothing executing, no wave expected: no completion will + // wake more callers, so flush what we have now. + return + } + fallback.Reset(gap) + } + } +} + +// dispatchCmds executes the drained stripe queues as one pipeline without +// constructing a Pipeline object: the queue slices go straight to the client's +// hook-wrapped pipeline processor (the exact entry Pipeline.Exec is wired to), +// so hooks and OTel behave identically while the per-batch Pipeline allocation, +// its append-growth reallocations and the per-command Process calls disappear. +// A single-stripe drain (every ordered shard, and any drain that found one +// non-empty stripe) passes its queue zero-copy; multi-stripe drains merge into +// one pooled slice. +// The batches stay OPEN throughout: completion happens at the caller's +// deferred closes, after the whole hook chain has returned. Hooks on the +// dispatch goroutine can still read results without deadlocking via the +// dispGid guard in await() (pre-next: the not-yet-executed view; post-next: +// the populated results), and — exactly like a plain pipeline — they may +// even adjust results before any waiter wakes. +// +// The innermost records whether execution actually happened. Two hook +// behaviours the chain's return value can carry are surfaced, both while the +// batches are still open (the callers' deferred closes run after this +// returns, so no waiter is reading yet): +// - short-circuit (hook returned without calling next): nothing set the +// commands' results — the chain's error, if any, is set +// on every command; +// - post-next verdict (exec ran, a hook still returned an error): applied +// to the commands ONLY when every one of them is error-free — the case +// where the hook's verdict would otherwise vanish entirely. A plain +// pipeline hands that verdict to the Exec caller without rewriting +// per-command results; with no Exec caller here, per-command errors +// recorded by the exec always win and are never overwritten. +func (ap *AutoPipeliner) dispatchCmds(ctx context.Context, queues [][]Cmder, total int) { + cmds := queues[0] + if len(queues) > 1 { + cmds = getQueueSlice(total) + for i := range queues { + cmds = append(cmds, queues[i]...) + } + } + // A command that forbids retries (today: the zero-copy reads, whose reply + // decodes into a caller buffer that a retry could not un-write) disables + // retries for the WHOLE slice it is dispatched in — see cmdsContainNoRetry. + // In a shared batch that would silently strip retries from unrelated + // callers' ordinary commands, so a mixed batch is dispatched as several + // pipelines instead of one. + // + // Split into CONTIGUOUS RUNS, in order, never into two policy groups: + // grouping would reorder the stream — a zero-copy read submitted before a + // SET to the same key would execute after it, so the read observes the new + // value on a face that promises submit order. Runs preserve every relative + // position while still keeping each dispatched slice policy-uniform (both + // findings by codex on #3942; the grouping bug was introduced by the first + // fix for the retry leak). + if runs := splitRetryRuns(cmds); runs != nil { + ap.dispatchSequential(ctx, runs) + if len(queues) > 1 { + putQueueSlice(cmds) + } + return + } + executed := false + chainErr := ap.pipeliner.withProcessPipelineHook(ctx, cmds, func(ctx context.Context, cmds []Cmder) error { + executed = true + return ap.pipeliner.processPipeline(ctx, cmds) + }) + // NOTE: a hook that returns nil WITHOUT calling next has short-circuited + // SUCCESSFULLY — it served the batch itself (a cache, a mock) and set the + // command values. Plain Pipeline/Client hooks are allowed to do exactly + // that, so no error is synthesized for it: doing so made a hook that works + // on a pipeline fail on an autopipelined batch (review finding by codex on + // #3942). Only the hook's own error propagates, below. + if chainErr != nil { + if !executed { + setCmdsErr(cmds, chainErr) + } else if cmdsFirstErr(cmds) == nil { + // Post-next error on an all-clean batch: the exec fully succeeded, + // so the error can only be the hook's own verdict — apply it. + // On a mixed batch it is applied to nothing: hooks conventionally + // return next's error (`err := next(...); return err`), so after a + // partial failure the chain error is presumed to be that echo, and + // stamping it on the commands that DID succeed would overwrite + // valid replies with their batchmates' failure. Exec-recorded + // per-command outcomes always win over a post-next rewrap. + setCmdsErr(cmds, chainErr) + } + } + if len(queues) > 1 { + putQueueSlice(cmds) + } +} + +// dispatchCmdsMaybeChunked dispatches a drained batch, splitting it into +// byte-bounded chunks when MaxBatchBytes is configured: each chunk is its own +// pipeline write+read cycle, so a batch of many large values becomes several +// bounded bursts instead of one huge write that can stall a constrained link +// past its deadline. The commands' batches still complete only after ALL +// chunks executed (the caller's deferred closes), exactly like an unchunked +// dispatch — chunking bounds the wire bursts, it does not change completion +// semantics. Each chunk runs the full hook chain, like consecutive pipelines. +func (ap *AutoPipeliner) dispatchCmdsMaybeChunked(ctx context.Context, queues [][]Cmder, total int) { + limit := int64(ap.config.MaxBatchBytes) + if limit <= 0 { + ap.dispatchCmds(ctx, queues, total) + return + } + + // Merge (borrowed from dispatchCmds's multi-queue path) so chunk + // boundaries can cross stripe queues. + cmds := queues[0] + merged := false + if len(queues) > 1 { + cmds = getQueueSlice(total) + for i := range queues { + cmds = append(cmds, queues[i]...) + } + merged = true + } + + // Cut the byte-bounded chunks, then hand the ordered sequence to the shared + // dispatcher — which stops after a chunk dies on a transport-class failure, + // so later commands cannot overtake a failed prefix (see + // dispatchSequential; the retry-policy runs go through the same helper). + chunks := make([][]Cmder, 0, 4) + start := 0 + var chunkBytes int64 + for i, cmd := range cmds { + chunkBytes += cmdApproxBytes(cmd) + if chunkBytes >= limit && i+1 > start { + chunks = append(chunks, cmds[start:i+1]) + start = i + 1 + chunkBytes = 0 + } + } + if start < len(cmds) { + chunks = append(chunks, cmds[start:]) + } + ap.dispatchSequential(ctx, chunks) + if merged { + putQueueSlice(cmds) + } +} + +// dispatchSequential dispatches an ORDERED sequence of sub-batches, stopping +// once one of them dies on a transport-class failure and failing the rest with +// that error. +// +// The stop is the same contract the unchunked path has: it fails or retries the +// batch as a UNIT, so in an ordered stream later commands must never overtake a +// prefix that died (retries exhausted, hook abort). Per-command redis errors +// (WRONGTYPE, nil) are normal outcomes and do not stop the sequence. +// +// Both callers that break a batch into ordered pieces — the MaxBatchBytes +// chunker and the retry-policy runs — go through here, because the first +// version of each got this wrong independently (review findings by codex on +// #3942). +func (ap *AutoPipeliner) dispatchSequential(ctx context.Context, groups [][]Cmder) { + var abortErr error + for _, group := range groups { + if len(group) == 0 { + continue + } + if abortErr != nil { + setCmdsErr(group, abortErr) + continue + } + ap.dispatchCmds(ctx, [][]Cmder{group}, len(group)) + for _, cmd := range group { + if err := cmd.rawErr(); err != nil && !isRedisError(err) { + abortErr = err + break + } + } + } +} + +// splitRetryRuns slices cmds into maximal CONTIGUOUS runs of one retry policy, +// preserving order: run i's commands all precede run i+1's, exactly as +// submitted. It returns nil when the whole batch is already policy-uniform — +// the overwhelmingly common case — so uniform batches allocate nothing and are +// dispatched as one pipeline. +// +// Runs are sub-slices of cmds, not copies, so they must be dispatched before +// cmds is recycled and must not be returned to the slice pool individually. +func splitRetryRuns(cmds []Cmder) [][]Cmder { + if len(cmds) < 2 { + return nil + } + first := cmds[0].NoRetry() + boundary := -1 + for i := 1; i < len(cmds); i++ { + if cmds[i].NoRetry() != first { + boundary = i + break + } + } + if boundary < 0 { + return nil // uniform: one dispatch, no split + } + runs := make([][]Cmder, 0, 4) + start := 0 + policy := first + for i := 1; i < len(cmds); i++ { + if p := cmds[i].NoRetry(); p != policy { + runs = append(runs, cmds[start:i]) + start = i + policy = p + } + } + return append(runs, cmds[start:]) +} + +// recoverDispatchPanic converts a panic on a dispatch goroutine (a hook or +// command-encoder panic inside Process/Exec) into per-command errors instead +// of crashing the process. On a plain client the same panic unwinds into the +// CALLER, who can recover; the engine's dispatch goroutines have no caller, +// so an unrecovered panic here would kill the whole program on behalf of one +// bad command. Registered LAST at each dispatch site so it runs FIRST on +// unwind (LIFO) — the errors are stamped before the deferred batch closes +// wake the waiters. setCmdsErr fills only commands without an error, so +// exec-recorded outcomes for commands that finished are preserved. +func recoverDispatchPanic(cmds ...[]Cmder) { + r := recover() + if r == nil { + return + } + err := fmt.Errorf("redis: autopipeline: panic during dispatch: %v", r) + for _, batch := range cmds { + setCmdsErr(batch, err) + } + internal.Logger.Printf(context.Background(), "autopipeline: recovered dispatch panic: %v\n%s", r, debug.Stack()) +} + +// flushBatchSlice takes the shard's currently-queued commands as one batch, +// swaps in a fresh batch for subsequent enqueues, and dispatches the taken +// batch. Completion is signalled by closing the batch's done channel once +// (waking every waiter in a single operation) rather than one channel send +// per command. +func (s *apShard) flushBatchSlice() { + ap := s.ap + + // Drain every stripe into one combined batch and roll fresh queues for the + // commands enqueued after this point. Striped enqueue spreads the hot + // mutex; one merged flush keeps the pipeline deep. accumulateBatch already + // bounds the total to roughly MaxBatchSize before we get here. + queues := make([][]Cmder, 0, len(s.stripes)) + batches := make([]*apBatch, 0, len(s.stripes)) + total := 0 + for i := range s.stripes { + st := &s.stripes[i] + // Skip provably-empty stripes without taking their mutex. Safe in + // THIS path only: an enqueue publishes queueLen under the stripe lock + // and wakes the flusher after unlocking, so a command that appears + // concurrently with this unlocked read is re-observed by the + // flusher's Len() loop or the buffered notify — the same protocol the + // flusher already relies on. The shutdown drain must keep locking + // unconditionally (see flushBatchSliceShutdown). + if st.queueLen.Load() == 0 { + continue + } + st.mu.Lock() + if len(st.queue) > 0 { + queues = append(queues, st.queue) + batches = append(batches, st.curBatch) + total += len(st.queue) + st.queue = getQueueSlice(ap.config.MaxBatchSize) + st.curBatch = newAPBatch() + st.queueLen.Store(0) + st.queueBytes.Store(0) + } + st.mu.Unlock() + } + if total == 0 { + return + } + + // Acquire a concurrency permit. The wait runs on a background context with + // a generous backstop deadline against a wedged semaphore: commands taken + // from the queue were already ACCEPTED, so a concurrent Close must not + // cancel them mid-acquire — Close's contract is to flush pending commands + // (it waits for this dispatch via wg/batchWg before tearing anything + // down). The backstop is deliberately well above both the default + // ReadTimeout and a maintnotifications relaxed window, so a legitimately + // slow batch (e.g. during a failover) holding a permit does not cause + // waiters to spuriously fail. + if !s.sem.TryAcquire() { + err := s.sem.Acquire(context.Background(), autoPipelinePermitBackstop, ErrAutoPipelineTimeout) + if err != nil { + // A permit not freeing within the backstop means the in-flight + // batch is wedged well past any configured timeout — leave an + // operator breadcrumb before failing the drained commands. + internal.Logger.Printf(context.Background(), + "redis: autopipeline: no batch permit after %s; failing %d queued commands", + autoPipelinePermitBackstop, total) + batchErr := err + for i := range queues { + for _, qc := range queues[i] { + qc.SetErr(batchErr) + } + batches[i].close() + putQueueSlice(queues[i]) + } + return + } + + // Wave merge. We took the queue and then waited a full batch round + // trip for the permit; callers whose replies landed just after our + // take re-submitted into the FRESH queue during that wait. Executing + // without them splits the group into two alternating waves — each + // observing two round trips, at half throughput — a state that is + // stable once entered (measured: p50 pinned at 2xRTT for entire runs + // at mid worker counts on a 52ms link). On the default window, let the + // wave of follow-ups land and fold it into this batch before + // executing, which merges the waves back into one batch per round + // trip. Explicit-delay configs keep their own timing. + if ap.config.MaxFlushDelay == 0 && !ap.config.AdaptiveDelay { + s.awaitExpectedArrivals(ap.config.MaxBatchSize) + for i := range s.stripes { + st := &s.stripes[i] + if st.queueLen.Load() == 0 { + continue + } + st.mu.Lock() + if len(st.queue) > 0 { + queues = append(queues, st.queue) + batches = append(batches, st.curBatch) + total += len(st.queue) + st.queue = getQueueSlice(ap.config.MaxBatchSize) + st.curBatch = newAPBatch() + st.queueLen.Store(0) + st.queueBytes.Store(0) + } + st.mu.Unlock() + } + } + } + + // Fast path for single command: skip the pipeline and Process directly, in + // its own goroutine. The dispatch MUST NOT run inline in the flusher: a + // synchronous Process blocks the flusher for a full round trip, and on a + // slow link a solo straggler then holds up an entire landed wave for one + // RTT — whose flush then delays the straggler's next command in turn, a + // stable phase-lock where everyone pays 2x RTT (measured: ~25% of runs on + // a 57ms link locked at exactly 2x RTT until perturbed). + // No expectedArrivals announcement: a single waiter waking is the + // lone-caller case, which must keep flushing immediately. + if total == 1 { + ap.batchWg.Add(1) + s.inFlight.Add(1) + go func() { + // Defer order matters: the batch close is registered BEFORE the + // permit release and inFlight decrement so it runs AFTER them + // (LIFO) — a woken lone caller's next command then observes an + // idle shard and takes the immediate-flush path instead of + // arming the silence-gap wait. + defer ap.batchWg.Done() + defer batches[0].close() + defer s.inFlight.Add(-1) + defer s.sem.Release() + defer putQueueSlice(queues[0]) + defer recoverDispatchPanic(queues[0]) + // Background for the same reason as the batch goroutine below: + // accepted commands execute even under a concurrent Close. + execStart := time.Now() + b := batches[0] + if !ap.blocking && ap.armSelfDeadlockGuard() { + b.dispGid.Store(curGoroutineID()) + } + solo := queues[0][0] + // Both faces run the user-hook chain via withProcessHook. The + // command records the CHAIN's final verdict — exactly what + // Client.Process does — before the deferred close wakes the + // waiter, so a hook that short-circuits, rewrites, or suppresses + // the error is honored. Hooks on this goroutine read the command + // deadlock-free via the dispGid guard stamped above. + // A successful short-circuit stays successful (see dispatchCmds). + err := ap.pipeliner.withProcessHook(context.Background(), solo, func(ctx context.Context, cmd Cmder) error { + return ap.pipeliner.process(ctx, cmd) + }) + solo.SetErr(err) + ap.observeBatchExec(time.Since(execStart)) + }() + return + } + + // Track this goroutine in the batchWg so Close() waits for it. + // IMPORTANT: Add to WaitGroup AFTER semaphore is acquired to avoid deadlock. + ap.batchWg.Add(1) + s.inFlight.Add(1) + go func() { + defer ap.batchWg.Done() + defer s.inFlight.Add(-1) + defer s.sem.Release() + // Signal completion with one close per taken stripe. Deferred so a + // panic in Process/Exec (e.g. a malformed command or encoder panic) + // still wakes every waiter in await() instead of hanging them forever; + // the closes run after Exec on the happy path, so results are + // populated first. + defer func() { + for i := range queues { + batches[i].close() + putQueueSlice(queues[i]) + } + }() + defer recoverDispatchPanic(queues...) + + // Execute on a background context: these commands were accepted before + // any concurrent Close, and Close waits for this goroutine (batchWg) + // before the client tears down its pools — cancelling here would + // error already-accepted commands while the shutdown sweep flushes + // later ones, an inverted outcome. The wire timeouts (Read/Write + // Timeout, or maintnotifications relaxed windows) still bound the + // execution; no per-batch timer is allocated. + ctx := context.Background() + + // The batches complete at the deferred closes, AFTER the whole hook + // chain has returned — so a hook's post-next verdict is honored and, + // like a plain pipeline, a hook may adjust results before any waiter + // wakes. Hooks on this goroutine read results deadlock-free via the + // dispGid guard in await() (armed below when hooks can exist). + if !ap.blocking && ap.armSelfDeadlockGuard() { + gid := curGoroutineID() + for i := range batches { + batches[i].dispGid.Store(gid) + } + } + + execStart := time.Now() + ap.dispatchCmdsMaybeChunked(ctx, queues, total) + ap.observeBatchExec(time.Since(execStart)) + + // Announce the expected arrivals BEFORE the deferred closes wake this + // batch's waiters, so the flusher knows the wave size the moment its + // first command lands (see expectedArrivals). + ap.expectedArrivals.Add(int64(total)) + }() +} + +// flushBatchSliceShutdown flushes commands during shutdown. +// Unlike flushBatchSlice, this doesn't use ap.ctx for semaphore acquisition +// because ap.ctx is already cancelled during shutdown. +// Executes synchronously to preserve command order. +func (s *apShard) flushBatchSliceShutdown() { + ap := s.ap + // Flush all remaining commands synchronously to preserve order. + // + // The loop condition is checked UNDER each stripe's lock (not via the + // unlocked s.Len()): a late enqueue appends to a stripe's queue and updates + // its queueLen under that stripe's mutex, so reading queueLen without the + // lock could miss a command that was just appended (seeing 0 and exiting + // while a command sits in the queue). Locking first makes "is the stripe + // empty?" and "take the stripe" atomic against that enqueue — this is what + // closes the lost-command race on Close. + for { + // Take every stripe's queue as one merged batch and roll fresh queues. + queues := make([][]Cmder, 0, len(s.stripes)) + batches := make([]*apBatch, 0, len(s.stripes)) + total := 0 + for i := range s.stripes { + st := &s.stripes[i] + st.mu.Lock() + if len(st.queue) > 0 { + queues = append(queues, st.queue) + batches = append(batches, st.curBatch) + total += len(st.queue) + st.queue = getQueueSlice(ap.config.MaxBatchSize) + st.curBatch = newAPBatch() + st.queueLen.Store(0) + st.queueBytes.Store(0) + } + st.mu.Unlock() + } + if total == 0 { + return + } + + // Serialize with any still-running in-flight batch: the shutdown drain + // used to bypass the per-shard permit, so under MaxConcurrentBatches:1 + // a drained command could execute CONCURRENTLY with the in-flight + // batch during Close and be observed out of order. Acquire the permit + // (bounded by the backstop, on a background context — ap.ctx is + // already cancelled here); if the backstop expires the permit holder + // is wedged and we proceed anyway rather than strand the commands. + acquired := s.sem.TryAcquire() + if !acquired { + acquired = s.sem.Acquire(context.Background(), autoPipelinePermitBackstop, ErrAutoPipelineTimeout) == nil + if !acquired { + internal.Logger.Printf(context.Background(), + "redis: autopipeline: no batch permit after %s during shutdown; flushing unserialized", + autoPipelinePermitBackstop) + } + } + + // Execute each batch in a func so close(batch.done) is deferred: a panic + // in Process/Exec still signals completion (waking await()) before it + // propagates, instead of leaving shutdown waiters hung. + func() { + if acquired { + defer s.sem.Release() + } + defer func() { + for i := range queues { + batches[i].close() + putQueueSlice(queues[i]) + } + }() + defer recoverDispatchPanic(queues...) + + // ap.ctx is already cancelled here (Close cancels it before draining), + // so use a fresh background context with no artificial deadline. The + // wire timeout is then governed by the connection's ReadTimeout / + // WriteTimeout — exactly like the normal flush path and a plain client + // Exec. Crucially this lets a relaxed timeout (set by maintnotifications + // during a failover/migration) take effect; a hardcoded short deadline + // here would cap that relaxed window and time out in-flight commands the + // relaxation was meant to protect. (A user who wants shutdown bounded + // sets ReadTimeout/WriteTimeout on the client, as for any command.) + if !ap.blocking && ap.armSelfDeadlockGuard() { + gid := curGoroutineID() + for i := range batches { + batches[i].dispGid.Store(gid) + } + } + ap.dispatchCmdsMaybeChunked(context.Background(), queues, total) + }() + } +} + +// Len returns the number of queued commands in this shard. +func (s *apShard) Len() int { + n := 0 + for i := range s.stripes { + n += int(s.stripes[i].queueLen.Load()) + } + return n +} + +// bytesFull reports whether the shard's queued payload volume has reached the +// configured MaxBatchBytes (false when the cap is disabled). Like the +// MaxBatchSize trigger it is soft: enqueues racing the check can overshoot. +func (s *apShard) bytesFull() bool { + limit := int64(s.ap.config.MaxBatchBytes) + if limit <= 0 { + return false + } + var n int64 + for i := range s.stripes { + n += s.stripes[i].queueBytes.Load() + if n >= limit { + return true + } + } + return false +} + +// cmdApproxBytes estimates a command's wire payload for MaxBatchBytes +// accounting: string/[]byte argument lengths plus a small fixed overhead per +// argument (type marker, length line, CRLFs). Exactness doesn't matter — the +// cap bounds burst size, it is not a protocol calculation. +func cmdApproxBytes(cmd Cmder) int64 { + const perArgOverhead = 16 + n := int64(0) + for _, a := range cmd.Args() { + switch v := a.(type) { + case string: + n += int64(len(v)) + case []byte: + n += int64(len(v)) + default: + n += 8 + } + n += perArgOverhead + } + return n +} + +// Len returns the current number of queued commands across all shards. +func (ap *AutoPipeliner) Len() int { + total := 0 + for _, s := range ap.shards { + total += s.Len() + } + return total +} + +// calculateDelay calculates the delay based on the given queue length (the +// caller's own shard, not the global total, so each shard tunes independently). +// Uses integer-only arithmetic for optimal performance (no float operations). +// Returns 0 if MaxFlushDelay is 0. +func (ap *AutoPipeliner) calculateDelay(queueLen int) time.Duration { + maxDelay := ap.config.MaxFlushDelay + if maxDelay == 0 { + return 0 + } + + // If adaptive delay is disabled, return fixed delay + if !ap.config.AdaptiveDelay { + return maxDelay + } + + if queueLen == 0 { + return 0 + } + + maxBatch := ap.config.MaxBatchSize + + // Use integer arithmetic to avoid float operations + // Calculate thresholds: 75%, 50%, 25% of maxBatch + // Multiply by 4 to avoid division: queueLen * 4 vs maxBatch * 3 (75%) + // + // Adaptive delay strategy: + // - ≥75% full: No delay (flush immediately to prevent overflow) + // - ≥50% full: 25% of max delay (queue filling up) + // - ≥25% full: 50% of max delay (moderate load) + // - <25% full: 100% of max delay (low load, maximize batching) + switch { + case queueLen*4 >= maxBatch*3: // queueLen >= 75% of maxBatch + return 0 // Flush immediately + case queueLen*2 >= maxBatch: // queueLen >= 50% of maxBatch + return maxDelay >> 2 // Divide by 4 using bit shift (faster) + case queueLen*4 >= maxBatch: // queueLen >= 25% of maxBatch + return maxDelay >> 1 // Divide by 2 using bit shift (faster) + default: + return maxDelay + } +} + +// Pipeline returns a new pipeline that uses the underlying pipeliner. +// This allows you to create a traditional pipeline from an autopipeliner. +func (ap *AutoPipeliner) Pipeline() Pipeliner { + return ap.pipeliner.Pipeline() +} + +// Pipelined executes a function in a pipeline context. +// This is a convenience method that creates a pipeline, executes the function, +// and returns the results. +func (ap *AutoPipeliner) Pipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { + return ap.pipeliner.Pipeline().Pipelined(ctx, fn) +} + +// TxPipelined executes a function in a transaction pipeline context. +// This is a convenience method that creates a transaction pipeline, executes the function, +// and returns the results. It delegates to the underlying client's TxPipeline. +func (ap *AutoPipeliner) TxPipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { + return ap.pipeliner.TxPipeline().Pipelined(ctx, fn) +} + +// TxPipeline returns a new transaction pipeline that uses the underlying pipeliner. +// This allows you to create a traditional transaction pipeline from an autopipeliner. +// It delegates to the underlying client's TxPipeline. +func (ap *AutoPipeliner) TxPipeline() Pipeliner { + return ap.pipeliner.TxPipeline() +} + +// validate AutoPipeliner implements Cmdable +var _ Cmdable = (*AutoPipeliner)(nil) diff --git a/backend/vendor/github.com/redis/go-redis/v9/autopipeline_bench_README.md b/backend/vendor/github.com/redis/go-redis/v9/autopipeline_bench_README.md new file mode 100644 index 00000000..40febdf9 --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/autopipeline_bench_README.md @@ -0,0 +1,128 @@ +# Autopipelining benchmarks + +These benchmarks validate the goal of autopipelining: batching concurrent +commands into pipelines cuts network round-trips and raises throughput, without +callers writing pipeline code. + +Two measurement rules keep every number honest: + +- **Throughput is measured on executed commands** — a command is counted only + after its result has been read (`.Result()` / `.Err()`), never when it is + merely queued. On the deferred face `ap.Set` returns immediately, so counting + calls would measure enqueue speed, not throughput. +- **Rates divide by the timed region, not the nominal window.** The + fixed-duration drivers keep draining whatever was in flight when the deadline + hit; that drain is part of the timed region, so `ops/sec` cannot be inflated + by work that finished after the window closed. + +## Running + +The benchmarks talk to a real Redis on `:6379` (they skip when none answers). +Start one first: + +```sh +make docker.start # or: docker run --rm -p 6379:6379 redis + +# the headline three-way throughput comparison: +go test -run '^$' -bench BenchmarkAutoPipelineThroughput -benchtime=1x . + +# everything: +go test -run '^$' -bench Benchmark -benchmem -benchtime=1x . +``` + +The throughput benchmarks run for a fixed wall-clock duration (~3s each) and +report `ops/sec`, so `-benchtime=1x` (one iteration) is correct for them. +**Do not pass a time-based `-benchtime`** (e.g. `-benchtime=5s`): the +fixed-duration drivers ignore `b.N`, so Go's framework would re-run the full +window geometrically trying to fill the time budget. + +The per-operation benchmarks (`BenchmarkDispatchPath`, +`BenchmarkAutoPipelineZeroCopy`) are the opposite: their ns/op and allocs/op +are only meaningful at the **default** `-benchtime` — at `-benchtime=1x` the +workers still issue a minimum window each, all billed to a single iteration. +(The "everything" command above also sweeps the repo's other root-package +benchmarks; that is harmless, just broader than this file.) + +## The headline benchmark: BenchmarkAutoPipelineThroughput + +Three ways to issue the same workload (2000 goroutines), each counting only +executed commands: + +1. **Normal** — a plain client; each `Set` is a blocking round-trip. Bounded by + Redis's non-pipelined ceiling (like `redis-benchmark` without `-P`). +2. **AutoPipelineBlocking** — the blocking face with a parallel-batch config: + `ap.Set(...)` blocks until executed, the same call shape as a normal client. + Only one command per caller is in flight, but the flusher batches across the + 2000 callers into deep pipelines. +3. **AutoPipelineWindowed** — the deferred face: each caller submits a window of + 200 commands, then reads the results. Keeps pipelines deepest. + +The `WindowedGET` variant repeats (3) with GET instead of SET: SET throughput is +server-bound (Redis's write processing), GET is cheaper on the server, so the +GET number shows the client machinery itself is not the limit. + +**Absolute numbers are machine- and load-dependent and vary a lot** — CPU +count, Redis's own ceiling, network path (loopback vs docker veth vs real +network), and noisy neighbors all move them by integer factors. The signal is +the WITHIN-RUN multiplier against the `Normal` baseline measured in the same +environment, plus `allocs/op` (which is exact and stable everywhere): + +| variant | vs Normal (same run) | +| ----------------------- | ---------------------- | +| Normal | 1x (the baseline) | +| AutoPipelineBlocking | ~10x | +| AutoPipelineWindowed | ~25-30x | +| AutoPipelineWindowedGET | above Windowed (reads) | + +As one concrete example: an Apple Silicon laptop with a loopback Redis puts +`Normal` around 80k ops/sec (so ~800k blocking, ~2.5M windowed); a 4-vCPU CI +runner with dockerized Redis lands near half that on the CPU-bound variants — +different absolutes, same multipliers and ordering. + +The autopipeline variants use an explicit parallel-batch config +(`MaxBatchSize: 300, MaxConcurrentBatches: 80, Unordered: true`) — **not the +ordered default**. The default (`MaxConcurrentBatches: 1`, +`DefaultAutoPipelineOptions` / `DefaultBlockingAutoPipelineOptions`) serializes +batch execution: blocking usage lands at roughly half the parallel-batch +multiplier, while windowed submission stays well into the tens-of-x even +ordered. + +## The other benchmarks + +- **BenchmarkIndividualCommands** — plain-client baseline: one blocking + round-trip per command across GOMAXPROCS workers. Its ns/op is your + environment's RTT floor; every other number is best read against it. +- **BenchmarkManualPipeline** — hand-built 100-deep `Pipeline().Exec()`, + sequential: the per-command cost of explicit pipelining (roughly a tenth + of a round-trip per command). The ceiling autopipelining approaches + without anyone writing pipeline code. +- **BenchmarkDispatchPath** — the engine's per-command dispatch cost with + honest `b.N` accounting: ns/op and allocs/op per executed command + (4 allocs/cmd on the submit path; unordered dispatch roughly halves the + ordered ns/op), plus the lone-command blocking fast path (~1 RTT). +- **BenchmarkFutureFace** — the typed future face on the ordered default + config: per-command reads (`InOrder`) vs windowed reads (`Window200`, + roughly 2x InOrder). +- **BenchmarkAutoPipelineSubmit** — the non-blocking `Submit` entry point, + windowed, on the ordered default; lands in the same band as + `FutureFace/Window200`. +- **BenchmarkAutoPipelineZeroCopy** — `GetToBuffer`/`SetFromBuffer` vs regular + `Get`/`Set` (Set+Get pairs): B/op drops ~10x at 4KiB and ~90x at 64KiB + (payloads decode into the caller's buffer instead of fresh strings), with + throughput at parity or better; allocs/op 10 vs 11. The B/op and allocs/op + ratios are environment-independent. +- **BenchmarkClusterAutoPipelineThroughput** — the same blocking/windowed + drivers against a local 3-master cluster (slot-routed shard batches keep + per-node pipelines deep; scales past the standalone numbers in the same + environment, with a wide run-order-dependent spread). Skips when no + cluster answers on `:16600-16602`. + +## What was deliberately removed + +Earlier revisions carried "tuning sweep" benchmarks (batch sizes, flush +delays, buffer sizes) whose numbers were dominated by the configured +`MaxFlushDelay` timer at low parallelism — every swept value reported the same +timer readout, which could only mislead someone tuning from them. They were +removed rather than fixed: `BenchmarkDispatchPath` and the throughput drivers +cover the engine's real knobs. Tune with your own workload shape; the engine's +defaults need no tuning to hit the numbers above. diff --git a/backend/vendor/github.com/redis/go-redis/v9/command.go b/backend/vendor/github.com/redis/go-redis/v9/command.go index ae0158b3..f0575c0e 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/command.go +++ b/backend/vendor/github.com/redis/go-redis/v9/command.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/redis/go-redis/v9/internal" @@ -155,6 +156,7 @@ const ( CmdTypeFTSearch CmdTypeTSTimestampValue CmdTypeTSTimestampValueSlice + CmdTypeTSNRangePivotRowSlice CmdTypeHotKeys CmdTypeIncrEXInt CmdTypeIncrEXFloat @@ -222,12 +224,24 @@ type Cmder interface { stepCount() int8 SetStepCount(int8) + // cachedSlot/setCachedSlot memoize the cluster slot so it is computed once + // (in the autopipeline shard router) and reused at pipeline-flush routing. + cachedSlot() (int, bool) + setCachedSlot(int) + readTimeout() *time.Duration readReply(rd *proto.Reader) error readRawReply(rd *proto.Reader) error SetErr(error) Err() error + // setReady marks a command as asynchronously pending (autopipeline async + // faces); await blocks the public accessors until it has executed; rawErr + // reads the error without awaiting (internal execution path). + setReady(*apBatch) + await() + rawErr() error + // NoRetry returns true if the command should not be retried on failure. // Commands that write directly to an io.Writer should return true since // partial writes cannot be undone on retry. @@ -239,7 +253,8 @@ type Cmder interface { func setCmdsErr(cmds []Cmder, e error) { for _, cmd := range cmds { - if cmd.Err() == nil { + // rawErr: this runs on the execution path; never await here. + if cmd.rawErr() == nil { cmd.SetErr(e) } } @@ -247,7 +262,8 @@ func setCmdsErr(cmds []Cmder, e error) { func cmdsFirstErr(cmds []Cmder) error { for _, cmd := range cmds { - if err := cmd.Err(); err != nil { + // rawErr: this runs on the execution path; never await here. + if err := cmd.rawErr(); err != nil { return err } } @@ -294,6 +310,14 @@ func cmdFirstKeyPosWithInfo(cmd Cmder, info *CommandInfo) int { return 0 } + // Module commands registered keyless in the static policy table (e.g. + // ft.aliaslist) route as keyless even while the command-info cache is + // cold, so the first calls of a process don't hash a non-key argument + // (such as an index name) into a slot. + if defaultPolicyKeyless(name) { + return 0 + } + switch name { case "eval", "evalsha", "eval_ro", "evalsha_ro": if cmd.stringArg(2) != "0" { @@ -328,7 +352,7 @@ func cmdString(cmd Cmder, val interface{}) string { b = internal.AppendArg(b, arg) } - if err := cmd.Err(); err != nil { + if err := cmd.rawErr(); err != nil { b = append(b, ": "...) b = append(b, err.Error()...) } else if val != nil { @@ -350,6 +374,90 @@ type baseCmd struct { rawVal interface{} _readTimeout *time.Duration cmdType CmdType + // slotCache memoizes the cluster slot once computed, so the cluster + // autopipeline shard router and the pipeline flush router don't each + // recompute it. 0 = not computed; it stores slot+1 so a real slot of 0 is + // distinguishable from unset. A plain field is safe by construction: it + // is written at most once, on the submitting goroutine BEFORE the command + // is published to a stripe queue (the stripe mutex is the happens-before + // edge to the flusher that later reads it). Do not write it from any + // other point in the command's life. + slotCache uint16 + + // ready, when non-nil, is the batch whose done channel closes once the + // command has executed. It is set only by the deferred (async) + // autopipeliner, which hands the command back to the caller before it + // runs. The public result accessors (Err/Val/Result/String) call await + // so they transparently block until execution; internal execution-path + // reads use rawErr to avoid awaiting the very batch they are producing + // (formatting included: cmdString reads rawErr and receives the value + // snapshot from its caller, so String methods await BEFORE reading their + // val field — otherwise formatting an in-flight async command would race + // with reply processing). ready stays + // nil for ordinary synchronous commands, whose accessors never block. + ready atomic.Pointer[apBatch] +} + +// setReady publishes the batch gating this command's result accessors. The +// field is atomic, NOT lock-ordered with the enqueue: a dispatch-side hook +// racing this store simply reads nil and takes the non-blocking path — the +// correct "not executed yet" view — while the setting goroutine always sees +// its own store before it awaits. +func (cmd *baseCmd) setReady(b *apBatch) { cmd.ready.Store(b) } + +// await blocks until an asynchronously-submitted command has executed. It is a +// single nil-pointer load for synchronous commands, so the common path stays +// allocation- and contention-free. +func (cmd *baseCmd) await() { + b := cmd.ready.Load() + if b == nil { + return + } + select { + case <-b.done: + return + default: + } + if b.isExecutorGoroutine() { + // A hook on one of the batch's own executor goroutines (the + // dispatcher, or a cluster per-node executor) is reading this + // command's result BEFORE next() has executed it. Blocking would + // self-deadlock (the batch completes only after that goroutine + // returns); return the not-yet-executed state instead — the same + // view a plain pipeline hook has before next(). + return + } + <-b.done +} + +// rawErr returns the command error WITHOUT awaiting. The internal +// execution/serialization path (setCmdsErr, cmdsFirstErr, and the cmdString +// formatter — public String methods await before calling it) uses it so that +// reading errors while a batch is being executed does not deadlock on the +// batch's own completion signal. +func (cmd *baseCmd) rawErr() error { return cmd.err } + +// readyBatch exposes the deferred-face batch gating this command (nil for +// synchronous commands) to the cluster fan-out, which registers its per-node +// goroutines as executors of every batch they carry. +func (cmd *baseCmd) readyBatch() *apBatch { return cmd.ready.Load() } + +// resultReady reports whether the command's result can be read WITHOUT +// blocking: either it never rode the deferred autopipeline face (no gating +// batch) or that batch has already completed. Post-execution bookkeeping in +// the command wrappers — the OTel metric emissions — consults it so that +// enabling telemetry cannot turn a deferred submission into a blocking call. +func (cmd *baseCmd) resultReady() bool { + b := cmd.ready.Load() + if b == nil { + return true + } + select { + case <-b.done: + return true + default: + return false + } } var _ Cmder = (*Cmd)(nil) @@ -389,6 +497,11 @@ func (cmd *baseCmd) stringArg(pos int) string { switch v := arg.(type) { case string: return v + case *string: + if v == nil { + return "" + } + return *v case []byte: return string(v) default: @@ -405,6 +518,21 @@ func (cmd *baseCmd) SetFirstKeyPos(keyPos int8) { cmd.keyPos = keyPos } +// cachedSlot returns the cached cluster slot and whether one was set. +func (cmd *baseCmd) cachedSlot() (int, bool) { + if cmd.slotCache == 0 { + return 0, false + } + return int(cmd.slotCache - 1), true +} + +// setCachedSlot stores the computed cluster slot (0..16383) for reuse. +func (cmd *baseCmd) setCachedSlot(slot int) { + if slot >= 0 && slot < 16384 { + cmd.slotCache = uint16(slot + 1) + } +} + func (cmd *baseCmd) stepCount() int8 { return cmd._stepCount } @@ -418,6 +546,7 @@ func (cmd *baseCmd) SetErr(e error) { } func (cmd *baseCmd) Err() error { + cmd.await() return cmd.err } @@ -488,6 +617,7 @@ func NewCmd(ctx context.Context, args ...interface{}) *Cmd { } func (cmd *Cmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -496,14 +626,17 @@ func (cmd *Cmd) SetVal(val interface{}) { } func (cmd *Cmd) Val() interface{} { + cmd.await() return cmd.val } func (cmd *Cmd) Result() (interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *Cmd) Text() (string, error) { + cmd.await() if cmd.err != nil { return "", cmd.err } @@ -521,6 +654,7 @@ func toString(val interface{}) (string, error) { } func (cmd *Cmd) Int() (int, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -536,6 +670,7 @@ func (cmd *Cmd) Int() (int, error) { } func (cmd *Cmd) Int64() (int64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -555,6 +690,7 @@ func toInt64(val interface{}) (int64, error) { } func (cmd *Cmd) Uint64() (uint64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -574,6 +710,7 @@ func toUint64(val interface{}) (uint64, error) { } func (cmd *Cmd) Float32() (float32, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -597,6 +734,7 @@ func toFloat32(val interface{}) (float32, error) { } func (cmd *Cmd) Float64() (float64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -616,6 +754,7 @@ func toFloat64(val interface{}) (float64, error) { } func (cmd *Cmd) Bool() (bool, error) { + cmd.await() if cmd.err != nil { return false, cmd.err } @@ -637,6 +776,7 @@ func toBool(val interface{}) (bool, error) { } func (cmd *Cmd) Slice() ([]interface{}, error) { + cmd.await() if cmd.err != nil { return nil, cmd.err } @@ -787,18 +927,22 @@ func (cmd *RawCmd) SetVal(val []byte) { } func (cmd *RawCmd) Val() []byte { + cmd.await() return cmd.val } func (cmd *RawCmd) Result() ([]byte, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *RawCmd) Bytes() ([]byte, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *RawCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -846,14 +990,17 @@ func (cmd *RawWriteToCmd) SetVal(written int64) { } func (cmd *RawWriteToCmd) Val() int64 { + cmd.await() return cmd.written } func (cmd *RawWriteToCmd) Result() (int64, error) { + cmd.await() return cmd.written, cmd.err } func (cmd *RawWriteToCmd) String() string { + cmd.await() return cmdString(cmd, cmd.written) } @@ -914,20 +1061,24 @@ func (cmd *ZeroCopyStringCmd) SetVal(n int) { } func (cmd *ZeroCopyStringCmd) Val() int { + cmd.await() return cmd.n } // Result returns the number of bytes read and any error. func (cmd *ZeroCopyStringCmd) Result() (int, error) { + cmd.await() return cmd.n, cmd.err } // Bytes returns the slice of the user-provided buffer containing the read data. func (cmd *ZeroCopyStringCmd) Bytes() []byte { + cmd.await() return cmd.buf[:cmd.n] } func (cmd *ZeroCopyStringCmd) String() string { + cmd.await() return cmdString(cmd, cmd.n) } @@ -1012,20 +1163,24 @@ func (cmd *SliceCmd) SetVal(val []interface{}) { } func (cmd *SliceCmd) Val() []interface{} { + cmd.await() return cmd.val } func (cmd *SliceCmd) Result() ([]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *SliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } // Scan scans the results from the map into a destination struct. The map keys // are matched in the Redis struct fields by the `redis:"field"` tag. func (cmd *SliceCmd) Scan(dst interface{}) error { + cmd.await() if cmd.err != nil { return cmd.err } @@ -1085,18 +1240,22 @@ func (cmd *StatusCmd) SetVal(val string) { } func (cmd *StatusCmd) Val() string { + cmd.await() return cmd.val } func (cmd *StatusCmd) Result() (string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StatusCmd) Bytes() ([]byte, error) { + cmd.await() return util.StringToBytes(cmd.val), cmd.err } func (cmd *StatusCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1137,18 +1296,22 @@ func (cmd *IntCmd) SetVal(val int64) { } func (cmd *IntCmd) Val() int64 { + cmd.await() return cmd.val } func (cmd *IntCmd) Result() (int64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *IntCmd) Uint64() (uint64, error) { + cmd.await() return uint64(cmd.val), cmd.err } func (cmd *IntCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1187,14 +1350,17 @@ func (cmd *UintCmd) SetVal(val uint64) { } func (cmd *UintCmd) Val() uint64 { + cmd.await() return cmd.val } func (cmd *UintCmd) Result() (uint64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *UintCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1247,14 +1413,17 @@ func (cmd *DigestCmd) SetVal(val uint64) { } func (cmd *DigestCmd) Val() uint64 { + cmd.await() return cmd.val } func (cmd *DigestCmd) Result() (uint64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *DigestCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1304,14 +1473,17 @@ func (cmd *IntSliceCmd) SetVal(val []int64) { } func (cmd *IntSliceCmd) Val() []int64 { + cmd.await() return cmd.val } func (cmd *IntSliceCmd) Result() ([]int64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *IntSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1322,8 +1494,13 @@ func (cmd *IntSliceCmd) readReply(rd *proto.Reader) error { } cmd.val = make([]int64, n) for i := 0; i < len(cmd.val); i++ { - if cmd.val[i], err = rd.ReadInt(); err != nil { + switch num, err := rd.ReadInt(); { + case err == Nil: + cmd.val[i] = 0 + case err != nil: return err + default: + cmd.val[i] = num } } return nil @@ -1364,14 +1541,17 @@ func (cmd *UintSliceCmd) SetVal(val []uint64) { } func (cmd *UintSliceCmd) Val() []uint64 { + cmd.await() return cmd.val } func (cmd *UintSliceCmd) Result() ([]uint64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *UintSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1382,8 +1562,13 @@ func (cmd *UintSliceCmd) readReply(rd *proto.Reader) error { } cmd.val = make([]uint64, n) for i := range cmd.val { - if cmd.val[i], err = rd.ReadUint(); err != nil { + switch num, err := rd.ReadUint(); { + case err == Nil: + cmd.val[i] = 0 + case err != nil: return err + default: + cmd.val[i] = num } } return nil @@ -1428,14 +1613,17 @@ func (cmd *DurationCmd) SetVal(val time.Duration) { } func (cmd *DurationCmd) Val() time.Duration { + cmd.await() return cmd.val } func (cmd *DurationCmd) Result() (time.Duration, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *DurationCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1488,14 +1676,17 @@ func (cmd *TimeCmd) SetVal(val time.Time) { } func (cmd *TimeCmd) Val() time.Time { + cmd.await() return cmd.val } func (cmd *TimeCmd) Result() (time.Time, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *TimeCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1547,14 +1738,17 @@ func (cmd *BoolCmd) SetVal(val bool) { } func (cmd *BoolCmd) Val() bool { + cmd.await() return cmd.val } func (cmd *BoolCmd) Result() (bool, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *BoolCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1602,18 +1796,22 @@ func (cmd *StringCmd) SetVal(val string) { } func (cmd *StringCmd) Val() string { + cmd.await() return cmd.val } func (cmd *StringCmd) Result() (string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StringCmd) Bytes() ([]byte, error) { + cmd.await() return util.StringToBytes(cmd.val), cmd.err } func (cmd *StringCmd) Bool() (bool, error) { + cmd.await() if cmd.err != nil { return false, cmd.err } @@ -1621,6 +1819,7 @@ func (cmd *StringCmd) Bool() (bool, error) { } func (cmd *StringCmd) Int() (int, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1628,6 +1827,7 @@ func (cmd *StringCmd) Int() (int, error) { } func (cmd *StringCmd) Int64() (int64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1635,6 +1835,7 @@ func (cmd *StringCmd) Int64() (int64, error) { } func (cmd *StringCmd) Uint64() (uint64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1642,6 +1843,7 @@ func (cmd *StringCmd) Uint64() (uint64, error) { } func (cmd *StringCmd) Float32() (float32, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1653,6 +1855,7 @@ func (cmd *StringCmd) Float32() (float32, error) { } func (cmd *StringCmd) Float64() (float64, error) { + cmd.await() if cmd.err != nil { return 0, cmd.err } @@ -1660,6 +1863,7 @@ func (cmd *StringCmd) Float64() (float64, error) { } func (cmd *StringCmd) Time() (time.Time, error) { + cmd.await() if cmd.err != nil { return time.Time{}, cmd.err } @@ -1667,6 +1871,7 @@ func (cmd *StringCmd) Time() (time.Time, error) { } func (cmd *StringCmd) Scan(val interface{}) error { + cmd.await() if cmd.err != nil { return cmd.err } @@ -1674,6 +1879,7 @@ func (cmd *StringCmd) Scan(val interface{}) error { } func (cmd *StringCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1714,14 +1920,17 @@ func (cmd *FloatCmd) SetVal(val float64) { } func (cmd *FloatCmd) Val() float64 { + cmd.await() return cmd.val } func (cmd *FloatCmd) Result() (float64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FloatCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1762,14 +1971,17 @@ func (cmd *FloatSliceCmd) SetVal(val []float64) { } func (cmd *FloatSliceCmd) Val() []float64 { + cmd.await() return cmd.val } func (cmd *FloatSliceCmd) Result() ([]float64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FloatSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1830,18 +2042,22 @@ func (cmd *StringSliceCmd) SetVal(val []string) { } func (cmd *StringSliceCmd) Val() []string { + cmd.await() return cmd.val } func (cmd *StringSliceCmd) Result() ([]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StringSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *StringSliceCmd) ScanSlice(container interface{}) error { + cmd.await() return proto.ScanSlice(cmd.val, container) } @@ -1902,14 +2118,17 @@ func (cmd *StringSliceSliceCmd) SetVal(val [][]string) { } func (cmd *StringSliceSliceCmd) Val() [][]string { + cmd.await() return cmd.val } func (cmd *StringSliceSliceCmd) Result() ([][]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StringSliceSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1987,14 +2206,17 @@ func (cmd *KeyValueSliceCmd) SetVal(val []KeyValue) { } func (cmd *KeyValueSliceCmd) Val() []KeyValue { + cmd.await() return cmd.val } func (cmd *KeyValueSliceCmd) Result() ([]KeyValue, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *KeyValueSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2030,6 +2252,9 @@ func (cmd *KeyValueSliceCmd) readReply(rd *proto.Reader) error { // nolint:dupl if array { cmd.val = make([]KeyValue, n) } else { + if n%2 != 0 { + return fmt.Errorf("redis: got %d elements in the key-value array, wanted a multiple of 2", n) + } cmd.val = make([]KeyValue, n/2) } @@ -2089,14 +2314,17 @@ func (cmd *BoolSliceCmd) SetVal(val []bool) { } func (cmd *BoolSliceCmd) Val() []bool { + cmd.await() return cmd.val } func (cmd *BoolSliceCmd) Result() ([]bool, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *BoolSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2107,8 +2335,13 @@ func (cmd *BoolSliceCmd) readReply(rd *proto.Reader) error { } cmd.val = make([]bool, n) for i := 0; i < len(cmd.val); i++ { - if cmd.val[i], err = rd.ReadBool(); err != nil { + switch b, err := rd.ReadBool(); { + case err == Nil: + cmd.val[i] = false + case err != nil: return err + default: + cmd.val[i] = b } } return nil @@ -2147,6 +2380,7 @@ func NewMapStringStringCmd(ctx context.Context, args ...interface{}) *MapStringS } func (cmd *MapStringStringCmd) Val() map[string]string { + cmd.await() return cmd.val } @@ -2155,16 +2389,19 @@ func (cmd *MapStringStringCmd) SetVal(val map[string]string) { } func (cmd *MapStringStringCmd) Result() (map[string]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringStringCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } // Scan scans the results from the map into a destination struct. The map keys // are matched in the Redis struct fields by the `redis:"field"` tag. func (cmd *MapStringStringCmd) Scan(dest interface{}) error { + cmd.await() if cmd.err != nil { return cmd.err } @@ -2245,14 +2482,17 @@ func (cmd *MapStringIntCmd) SetVal(val map[string]int64) { } func (cmd *MapStringIntCmd) Val() map[string]int64 { + cmd.await() return cmd.val } func (cmd *MapStringIntCmd) Result() (map[string]int64, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringIntCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2309,6 +2549,7 @@ func NewMapStringSliceInterfaceCmd(ctx context.Context, args ...interface{}) *Ma } func (cmd *MapStringSliceInterfaceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2317,10 +2558,12 @@ func (cmd *MapStringSliceInterfaceCmd) SetVal(val map[string][]interface{}) { } func (cmd *MapStringSliceInterfaceCmd) Result() (map[string][]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringSliceInterfaceCmd) Val() map[string][]interface{} { + cmd.await() return cmd.val } @@ -2384,6 +2627,11 @@ func (cmd *MapStringSliceInterfaceCmd) readReply(rd *proto.Reader) (err error) { cmd.val[key] = append(cmd.val[key], data) } } + default: + // Any other reply type leaves the peeked frame unread. Returning nil + // here would put the connection back in the pool with those bytes + // buffered, so the next command reads them as its own reply. + return fmt.Errorf("redis: can't parse map-string-slice-interface reply: unexpected type %c", readType) } return nil @@ -2432,14 +2680,17 @@ func (cmd *StringStructMapCmd) SetVal(val map[string]struct{}) { } func (cmd *StringStructMapCmd) Val() map[string]struct{} { + cmd.await() return cmd.val } func (cmd *StringStructMapCmd) Result() (map[string]struct{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *StringStructMapCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2507,14 +2758,17 @@ func (cmd *XMessageSliceCmd) SetVal(val []XMessage) { } func (cmd *XMessageSliceCmd) Val() []XMessage { + cmd.await() return cmd.val } func (cmd *XMessageSliceCmd) Result() ([]XMessage, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XMessageSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2653,14 +2907,17 @@ func (cmd *XStreamSliceCmd) SetVal(val []XStream) { } func (cmd *XStreamSliceCmd) Val() []XStream { + cmd.await() return cmd.val } func (cmd *XStreamSliceCmd) Result() ([]XStream, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XStreamSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2757,14 +3014,17 @@ func (cmd *XPendingCmd) SetVal(val *XPending) { } func (cmd *XPendingCmd) Val() *XPending { + cmd.await() return cmd.val } func (cmd *XPendingCmd) Result() (*XPending, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XPendingCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2862,14 +3122,17 @@ func (cmd *XPendingExtCmd) SetVal(val []XPendingExt) { } func (cmd *XPendingExtCmd) Val() []XPendingExt { + cmd.await() return cmd.val } func (cmd *XPendingExtCmd) Result() ([]XPendingExt, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XPendingExtCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2946,14 +3209,17 @@ func (cmd *XAutoClaimCmd) SetVal(val []XMessage, start string) { } func (cmd *XAutoClaimCmd) Val() (messages []XMessage, start string) { + cmd.await() return cmd.val, cmd.start } func (cmd *XAutoClaimCmd) Result() (messages []XMessage, start string, err error) { + cmd.await() return cmd.val, cmd.start, cmd.err } func (cmd *XAutoClaimCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3040,14 +3306,17 @@ func (cmd *XAutoClaimWithDeletedCmd) SetVal(val []XMessage, start string, delete } func (cmd *XAutoClaimWithDeletedCmd) Val() (messages []XMessage, start string, deletedIDs []string) { + cmd.await() return cmd.val, cmd.start, cmd.deletedIDs } func (cmd *XAutoClaimWithDeletedCmd) Result() (messages []XMessage, start string, deletedIDs []string, err error) { + cmd.await() return cmd.val, cmd.start, cmd.deletedIDs, cmd.err } func (cmd *XAutoClaimWithDeletedCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3151,14 +3420,17 @@ func (cmd *XAutoClaimJustIDCmd) SetVal(val []string, start string) { } func (cmd *XAutoClaimJustIDCmd) Val() (ids []string, start string) { + cmd.await() return cmd.val, cmd.start } func (cmd *XAutoClaimJustIDCmd) Result() (ids []string, start string, err error) { + cmd.await() return cmd.val, cmd.start, cmd.err } func (cmd *XAutoClaimJustIDCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3247,14 +3519,17 @@ func (cmd *XInfoConsumersCmd) SetVal(val []XInfoConsumer) { } func (cmd *XInfoConsumersCmd) Val() []XInfoConsumer { + cmd.await() return cmd.val } func (cmd *XInfoConsumersCmd) Result() ([]XInfoConsumer, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XInfoConsumersCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3353,14 +3628,17 @@ func (cmd *XInfoGroupsCmd) SetVal(val []XInfoGroup) { } func (cmd *XInfoGroupsCmd) Val() []XInfoGroup { + cmd.await() return cmd.val } func (cmd *XInfoGroupsCmd) Result() ([]XInfoGroup, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XInfoGroupsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3491,14 +3769,17 @@ func (cmd *XInfoStreamCmd) SetVal(val *XInfoStream) { } func (cmd *XInfoStreamCmd) Val() *XInfoStream { + cmd.await() return cmd.val } func (cmd *XInfoStreamCmd) Result() (*XInfoStream, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XInfoStreamCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3718,14 +3999,17 @@ func (cmd *XInfoStreamFullCmd) SetVal(val *XInfoStreamFull) { } func (cmd *XInfoStreamFullCmd) Val() *XInfoStreamFull { + cmd.await() return cmd.val } func (cmd *XInfoStreamFullCmd) Result() (*XInfoStreamFull, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *XInfoStreamFullCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4100,14 +4384,17 @@ func (cmd *ZSliceCmd) SetVal(val []Z) { } func (cmd *ZSliceCmd) Val() []Z { + cmd.await() return cmd.val } func (cmd *ZSliceCmd) Result() ([]Z, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ZSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4132,6 +4419,9 @@ func (cmd *ZSliceCmd) readReply(rd *proto.Reader) error { // nolint:dupl if array { cmd.val = make([]Z, n) } else { + if n%2 != 0 { + return fmt.Errorf("redis: got %d elements in the sorted set array, wanted a multiple of 2", n) + } cmd.val = make([]Z, n/2) } @@ -4191,14 +4481,17 @@ func (cmd *ZWithKeyCmd) SetVal(val *ZWithKey) { } func (cmd *ZWithKeyCmd) Val() *ZWithKey { + cmd.await() return cmd.val } func (cmd *ZWithKeyCmd) Result() (*ZWithKey, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ZWithKeyCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4268,14 +4561,17 @@ func (cmd *ScanCmd) SetVal(page []string, cursor uint64) { } func (cmd *ScanCmd) Val() (keys []string, cursor uint64) { + cmd.await() return cmd.page, cmd.cursor } func (cmd *ScanCmd) Result() (keys []string, cursor uint64, err error) { + cmd.await() return cmd.page, cmd.cursor, cmd.err } func (cmd *ScanCmd) String() string { + cmd.await() return cmdString(cmd, cmd.page) } @@ -4362,14 +4658,17 @@ func (cmd *ClusterSlotsCmd) SetVal(val []ClusterSlot) { } func (cmd *ClusterSlotsCmd) Val() []ClusterSlot { + cmd.await() return cmd.val } func (cmd *ClusterSlotsCmd) Result() ([]ClusterSlot, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ClusterSlotsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4585,14 +4884,17 @@ func (cmd *GeoLocationCmd) SetVal(locations []GeoLocation) { } func (cmd *GeoLocationCmd) Val() []GeoLocation { + cmd.await() return cmd.locations } func (cmd *GeoLocationCmd) Result() ([]GeoLocation, error) { + cmd.await() return cmd.locations, cmd.err } func (cmd *GeoLocationCmd) String() string { + cmd.await() return cmdString(cmd, cmd.locations) } @@ -4794,14 +5096,17 @@ func (cmd *GeoSearchLocationCmd) SetVal(val []GeoLocation) { } func (cmd *GeoSearchLocationCmd) Val() []GeoLocation { + cmd.await() return cmd.val } func (cmd *GeoSearchLocationCmd) Result() ([]GeoLocation, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *GeoSearchLocationCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -4812,11 +5117,28 @@ func (cmd *GeoSearchLocationCmd) readReply(rd *proto.Reader) error { } cmd.val = make([]GeoLocation, n) + // Each element is an array of [name, ...] whose minimum length is set by + // the requested WITH flags. Entries shorter than that would make the + // parser read into the next reply; extra elements are drained below so a + // longer entry (e.g. from a newer server) can't leave frames on the wire. + withLen := 1 + if cmd.opt.WithDist { + withLen++ + } + if cmd.opt.WithHash { + withLen++ + } + if cmd.opt.WithCoord { + withLen++ + } for i := 0; i < n; i++ { - _, err = rd.ReadArrayLen() + nn, err := rd.ReadArrayLen() if err != nil { return err } + if nn < withLen { + return fmt.Errorf("redis: got %d elements in GEOSEARCH reply, expected at least %d", nn, withLen) + } var loc GeoLocation @@ -4849,6 +5171,11 @@ func (cmd *GeoSearchLocationCmd) readReply(rd *proto.Reader) error { return err } } + for j := withLen; j < nn; j++ { + if err := rd.DiscardNext(); err != nil { + return err + } + } cmd.val[i] = loc } @@ -4919,14 +5246,17 @@ func (cmd *GeoPosCmd) SetVal(val []*GeoPos) { } func (cmd *GeoPosCmd) Val() []*GeoPos { + cmd.await() return cmd.val } func (cmd *GeoPosCmd) Result() ([]*GeoPos, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *GeoPosCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5021,14 +5351,17 @@ func (cmd *CommandsInfoCmd) SetVal(val map[string]*CommandInfo) { } func (cmd *CommandsInfoCmd) Val() map[string]*CommandInfo { + cmd.await() return cmd.val } func (cmd *CommandsInfoCmd) Result() (map[string]*CommandInfo, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *CommandsInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5205,6 +5538,11 @@ type cmdsInfoCache struct { once internal.Once refreshLock sync.RWMutex cmds map[string]*CommandInfo + // cmdsAtomic mirrors cmds for lock-free reads via Peek. cmds is only ever + // replaced wholesale (never mutated in place), so an atomic pointer load is a + // safe, contention-free read — Peek is on the hot per-command cluster routing + // path where the RWMutex.RLock showed up as a bottleneck under heavy load. + cmdsAtomic atomic.Pointer[map[string]*CommandInfo] } func newCmdsInfoCache(fn func(ctx context.Context) (map[string]*CommandInfo, error)) *cmdsInfoCache { @@ -5231,6 +5569,7 @@ func (c *cmdsInfoCache) Get(ctx context.Context) (map[string]*CommandInfo, error } c.cmds = lowerCmds + c.cmdsAtomic.Store(&lowerCmds) return nil }) return c.cmds, err @@ -5245,16 +5584,19 @@ func (c *cmdsInfoCache) Refresh() { // Peek returns the cached CommandInfo map without triggering a Redis round-trip. // Returns nil when the cache is cold; callers should fall back to other heuristics. -// Note: during the very first Get() (initial population) this call will block on -// the writer lock. After that, concurrent Peek() calls do not block each other. +// The read is lock-free (a single atomic load) and never blocks, even while a +// concurrent Get() is populating the cache — it simply returns nil until the +// first population publishes the map. // The returned map and its entries MUST NOT be mutated by the caller. func (c *cmdsInfoCache) Peek() map[string]*CommandInfo { if c == nil { return nil } - c.refreshLock.RLock() - defer c.refreshLock.RUnlock() - return c.cmds + // Lock-free read: cmds is replaced wholesale, never mutated in place. + if p := c.cmdsAtomic.Load(); p != nil { + return *p + } + return nil } // ------------------------------------------------------------------------------ @@ -5301,6 +5643,10 @@ type SlowLog struct { // https://redis.io/commands/slowlog#output-format ClientAddr string ClientName string + // CommandArgc is the command's total argument count (including the command + // name), emitted only by Redis 8.10 or greater. It may exceed len(Args) when + // the slow log truncates the stored arguments (slowlog-max-argc, default 32). + CommandArgc int64 } type SlowLogCmd struct { @@ -5326,14 +5672,17 @@ func (cmd *SlowLogCmd) SetVal(val []SlowLog) { } func (cmd *SlowLogCmd) Val() []SlowLog { + cmd.await() return cmd.val } func (cmd *SlowLogCmd) Result() ([]SlowLog, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *SlowLogCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5396,6 +5745,21 @@ func (cmd *SlowLogCmd) readReply(rd *proto.Reader) error { return err } } + + // Redis 8.10+ appends a 7th field: the command's total argument count. + if nn >= 7 { + if cmd.val[i].CommandArgc, err = rd.ReadInt(); err != nil { + return err + } + } + + // Drain any elements past the 7 this parser knows about so a server + // that declares a longer entry array doesn't leave frames on the wire. + for j := 7; j < nn; j++ { + if err = rd.DiscardNext(); err != nil { + return err + } + } } return nil @@ -5407,11 +5771,12 @@ func (cmd *SlowLogCmd) Clone() Cmder { val = make([]SlowLog, len(cmd.val)) for i, log := range cmd.val { val[i] = SlowLog{ - ID: log.ID, - Time: log.Time, - Duration: log.Duration, - ClientAddr: log.ClientAddr, - ClientName: log.ClientName, + ID: log.ID, + Time: log.Time, + Duration: log.Duration, + ClientAddr: log.ClientAddr, + ClientName: log.ClientName, + CommandArgc: log.CommandArgc, } if log.Args != nil { val[i].Args = make([]string, len(log.Args)) @@ -5455,14 +5820,17 @@ func (cmd *LatencyCmd) SetVal(val []Latency) { } func (cmd *LatencyCmd) Val() []Latency { + cmd.await() return cmd.val } func (cmd *LatencyCmd) Result() ([]Latency, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *LatencyCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5477,8 +5845,8 @@ func (cmd *LatencyCmd) readReply(rd *proto.Reader) error { if err != nil { return err } - if nn < 3 { - return fmt.Errorf("redis: got %d elements in latency get, expected at least 3", nn) + if nn < 4 { + return fmt.Errorf("redis: got %d elements in latency get, expected at least 4", nn) } if cmd.val[i].Name, err = rd.ReadString(); err != nil { return err @@ -5498,6 +5866,13 @@ func (cmd *LatencyCmd) readReply(rd *proto.Reader) error { return err } cmd.val[i].Max = time.Duration(maximum) * time.Millisecond + // Drain any elements beyond the 4 this parser reads so a server that + // declares a longer entry array can't leave frames on the wire. + for j := 4; j < nn; j++ { + if err = rd.DiscardNext(); err != nil { + return err + } + } } return nil } @@ -5570,14 +5945,17 @@ func (cmd *HotKeysCmd) SetVal(val *HotKeysResult) { } func (cmd *HotKeysCmd) Val() *HotKeysResult { + cmd.await() return cmd.val } func (cmd *HotKeysCmd) Result() (*HotKeysResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *HotKeysCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5690,6 +6068,14 @@ func (cmd *HotKeysCmd) readReply(rd *proto.Reader) error { result.ByNetBytes = parseHotKeysKeyEntries(v) } + // Only the first element of the outer array is parsed; drain the rest so a + // server that wraps more than one element doesn't leave frames on the wire. + for i := 1; i < arrayLen; i++ { + if err := rd.DiscardNext(); err != nil { + return err + } + } + cmd.val = result return nil } @@ -5776,14 +6162,17 @@ func (cmd *MapStringInterfaceCmd) SetVal(val map[string]interface{}) { } func (cmd *MapStringInterfaceCmd) Val() map[string]interface{} { + cmd.await() return cmd.val } func (cmd *MapStringInterfaceCmd) Result() (map[string]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringInterfaceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5855,14 +6244,17 @@ func (cmd *MapStringStringSliceCmd) SetVal(val []map[string]string) { } func (cmd *MapStringStringSliceCmd) Val() []map[string]string { + cmd.await() return cmd.val } func (cmd *MapStringStringSliceCmd) Result() ([]map[string]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringStringSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5933,6 +6325,7 @@ func NewMapMapStringInterfaceCmd(ctx context.Context, args ...interface{}) *MapM } func (cmd *MapMapStringInterfaceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -5941,10 +6334,12 @@ func (cmd *MapMapStringInterfaceCmd) SetVal(val map[string]interface{}) { } func (cmd *MapMapStringInterfaceCmd) Result() (map[string]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapMapStringInterfaceCmd) Val() map[string]interface{} { + cmd.await() return cmd.val } @@ -6032,14 +6427,17 @@ func (cmd *MapStringInterfaceSliceCmd) SetVal(val []map[string]interface{}) { } func (cmd *MapStringInterfaceSliceCmd) Val() []map[string]interface{} { + cmd.await() return cmd.val } func (cmd *MapStringInterfaceSliceCmd) Result() ([]map[string]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *MapStringInterfaceSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -6119,14 +6517,17 @@ func (cmd *KeyValuesCmd) SetVal(key string, val []string) { } func (cmd *KeyValuesCmd) Val() (string, []string) { + cmd.await() return cmd.key, cmd.val } func (cmd *KeyValuesCmd) Result() (string, []string, error) { + cmd.await() return cmd.key, cmd.val, cmd.err } func (cmd *KeyValuesCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -6195,14 +6596,17 @@ func (cmd *ZSliceWithKeyCmd) SetVal(key string, val []Z) { } func (cmd *ZSliceWithKeyCmd) Val() (string, []Z) { + cmd.await() return cmd.key, cmd.val } func (cmd *ZSliceWithKeyCmd) Result() (string, []Z, error) { + cmd.await() return cmd.key, cmd.val, cmd.err } func (cmd *ZSliceWithKeyCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -6230,6 +6634,9 @@ func (cmd *ZSliceWithKeyCmd) readReply(rd *proto.Reader) (err error) { if array { cmd.val = make([]Z, n) } else { + if n%2 != 0 { + return fmt.Errorf("redis: got %d elements in the sorted set array, wanted a multiple of 2", n) + } cmd.val = make([]Z, n/2) } @@ -6301,18 +6708,22 @@ func (cmd *FunctionListCmd) SetVal(val []Library) { } func (cmd *FunctionListCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *FunctionListCmd) Val() []Library { + cmd.await() return cmd.val } func (cmd *FunctionListCmd) Result() ([]Library, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FunctionListCmd) First() (*Library, error) { + cmd.await() if cmd.err != nil { return nil, cmd.err } @@ -6514,14 +6925,17 @@ func (cmd *FunctionStatsCmd) SetVal(val FunctionStats) { } func (cmd *FunctionStatsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *FunctionStatsCmd) Val() FunctionStats { + cmd.await() return cmd.val } func (cmd *FunctionStatsCmd) Result() (FunctionStats, error) { + cmd.await() return cmd.val, cmd.err } @@ -6615,11 +7029,18 @@ func (cmd *FunctionStatsCmd) readEngines(rd *proto.Reader) ([]Engine, error) { for i := 0; i < 2; i++ { key, err := rd.ReadString() + if err != nil { + return nil, err + } switch key { case "libraries_count": engine.LibrariesCount, err = rd.ReadInt() case "functions_count": engine.FunctionsCount, err = rd.ReadInt() + default: + // Unknown field: drain its value so the reader stays aligned + // with the rest of the reply. + err = rd.DiscardNext() } if err != nil { return nil, err @@ -6779,14 +7200,17 @@ func (cmd *LCSCmd) SetVal(val *LCSMatch) { } func (cmd *LCSCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *LCSCmd) Val() *LCSMatch { + cmd.await() return cmd.val } func (cmd *LCSCmd) Result() (*LCSMatch, error) { + cmd.await() return cmd.val, cmd.err } @@ -6827,6 +7251,12 @@ func (cmd *LCSCmd) readReply(rd *proto.Reader) (err error) { if lcs.Len, err = rd.ReadInt(); err != nil { return err } + default: + // Unknown field: drain its value so the reader stays aligned + // with the rest of the reply. + if err = rd.DiscardNext(); err != nil { + return err + } } } } @@ -6929,14 +7359,17 @@ func (cmd *KeyFlagsCmd) SetVal(val []KeyFlags) { } func (cmd *KeyFlagsCmd) Val() []KeyFlags { + cmd.await() return cmd.val } func (cmd *KeyFlagsCmd) Result() ([]KeyFlags, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *KeyFlagsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7032,14 +7465,17 @@ func (cmd *ClusterLinksCmd) SetVal(val []ClusterLink) { } func (cmd *ClusterLinksCmd) Val() []ClusterLink { + cmd.await() return cmd.val } func (cmd *ClusterLinksCmd) Result() ([]ClusterLink, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ClusterLinksCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7147,14 +7583,17 @@ func (cmd *ClusterShardsCmd) SetVal(val []ClusterShard) { } func (cmd *ClusterShardsCmd) Val() []ClusterShard { + cmd.await() return cmd.val } func (cmd *ClusterShardsCmd) Result() ([]ClusterShard, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ClusterShardsCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7307,14 +7746,17 @@ func (cmd *RankWithScoreCmd) SetVal(val RankScore) { } func (cmd *RankWithScoreCmd) Val() RankScore { + cmd.await() return cmd.val } func (cmd *RankWithScoreCmd) Result() (RankScore, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *RankWithScoreCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7469,14 +7911,17 @@ func (cmd *ClientInfoCmd) SetVal(val *ClientInfo) { } func (cmd *ClientInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *ClientInfoCmd) Val() *ClientInfo { + cmd.await() return cmd.val } func (cmd *ClientInfoCmd) Result() (*ClientInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -7727,14 +8172,17 @@ func (cmd *ACLLogCmd) SetVal(val []*ACLLogEntry) { } func (cmd *ACLLogCmd) Val() []*ACLLogEntry { + cmd.await() return cmd.val } func (cmd *ACLLogCmd) Result() ([]*ACLLogEntry, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ACLLogCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7908,14 +8356,17 @@ func (cmd *InfoCmd) SetVal(val map[string]map[string]string) { } func (cmd *InfoCmd) Val() map[string]map[string]string { + cmd.await() return cmd.val } func (cmd *InfoCmd) Result() (map[string]map[string]string, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *InfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -7955,6 +8406,7 @@ func (cmd *InfoCmd) readReply(rd *proto.Reader) error { } func (cmd *InfoCmd) Item(section, key string) string { + cmd.await() if cmd.val == nil { return "" } else if cmd.val[section] == nil { @@ -8012,6 +8464,7 @@ func newMonitorCmd(ctx context.Context, ch chan string) *MonitorCmd { } func (cmd *MonitorCmd) String() string { + cmd.await() return cmdString(cmd, nil) } @@ -8096,14 +8549,17 @@ func (cmd *VectorScoreSliceCmd) SetVal(val []VectorScore) { } func (cmd *VectorScoreSliceCmd) Val() []VectorScore { + cmd.await() return cmd.val } func (cmd *VectorScoreSliceCmd) Result() ([]VectorScore, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *VectorScoreSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -8180,14 +8636,17 @@ func (cmd *VectorScoreSliceSliceCmd) SetVal(val [][]VectorScore) { } func (cmd *VectorScoreSliceSliceCmd) Val() [][]VectorScore { + cmd.await() return cmd.val } func (cmd *VectorScoreSliceSliceCmd) Result() ([][]VectorScore, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *VectorScoreSliceSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -8307,14 +8766,17 @@ func (cmd *VectorAttribSliceCmd) SetVal(val []VectorAttrib) { } func (cmd *VectorAttribSliceCmd) Val() []VectorAttrib { + cmd.await() return cmd.val } func (cmd *VectorAttribSliceCmd) Result() ([]VectorAttrib, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *VectorAttribSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -8395,14 +8857,17 @@ func (cmd *VectorScoreAttribSliceCmd) SetVal(val []VectorScoreAttrib) { } func (cmd *VectorScoreAttribSliceCmd) Val() []VectorScoreAttrib { + cmd.await() return cmd.val } func (cmd *VectorScoreAttribSliceCmd) Result() ([]VectorScoreAttrib, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *VectorScoreAttribSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -8926,6 +9391,13 @@ func ExtractCommandValue(cmd interface{}) (interface{}, error) { }); ok { return tsTimestampValueSliceCmd.Val(), tsTimestampValueSliceCmd.Err() } + case CmdTypeTSNRangePivotRowSlice: + if tsNRangePivotRowSliceCmd, ok := cmd.(interface { + Val() []TSNRangePivotRow + Err() error + }); ok { + return tsNRangePivotRowSliceCmd.Val(), tsNRangePivotRowSliceCmd.Err() + } case CmdTypeStringSlice: if stringSliceCmd, ok := cmd.(interface { Val() []string @@ -9064,11 +9536,18 @@ func NewIncrEXIntCmd(ctx context.Context, args ...interface{}) *IncrEXIntCmd { } func (cmd *IncrEXIntCmd) SetVal(val IncrEXIntResult) { cmd.val = val } -func (cmd *IncrEXIntCmd) Val() IncrEXIntResult { return cmd.val } +func (cmd *IncrEXIntCmd) Val() IncrEXIntResult { + cmd.await() + return cmd.val +} func (cmd *IncrEXIntCmd) Result() (IncrEXIntResult, error) { + cmd.await() return cmd.val, cmd.err } -func (cmd *IncrEXIntCmd) String() string { return cmdString(cmd, cmd.val) } +func (cmd *IncrEXIntCmd) String() string { + cmd.await() + return cmdString(cmd, cmd.val) +} func (cmd *IncrEXIntCmd) readReply(rd *proto.Reader) error { if err := rd.ReadFixedArrayLen(2); err != nil { @@ -9118,11 +9597,18 @@ func NewIncrEXFloatCmd(ctx context.Context, args ...interface{}) *IncrEXFloatCmd } func (cmd *IncrEXFloatCmd) SetVal(val IncrEXFloatResult) { cmd.val = val } -func (cmd *IncrEXFloatCmd) Val() IncrEXFloatResult { return cmd.val } +func (cmd *IncrEXFloatCmd) Val() IncrEXFloatResult { + cmd.await() + return cmd.val +} func (cmd *IncrEXFloatCmd) Result() (IncrEXFloatResult, error) { + cmd.await() return cmd.val, cmd.err } -func (cmd *IncrEXFloatCmd) String() string { return cmdString(cmd, cmd.val) } +func (cmd *IncrEXFloatCmd) String() string { + cmd.await() + return cmdString(cmd, cmd.val) +} func (cmd *IncrEXFloatCmd) readReply(rd *proto.Reader) error { if err := rd.ReadFixedArrayLen(2); err != nil { @@ -9172,14 +9658,17 @@ func (cmd *AREntrySliceCmd) SetVal(val []AREntry) { } func (cmd *AREntrySliceCmd) Val() []AREntry { + cmd.await() return cmd.val } func (cmd *AREntrySliceCmd) Result() ([]AREntry, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *AREntrySliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } diff --git a/backend/vendor/github.com/redis/go-redis/v9/command_policy_resolver.go b/backend/vendor/github.com/redis/go-redis/v9/command_policy_resolver.go index da8c6d31..11fc5d1c 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/command_policy_resolver.go +++ b/backend/vendor/github.com/redis/go-redis/v9/command_policy_resolver.go @@ -109,6 +109,13 @@ var defaultPolicies = map[module]map[commandName]*routing.CommandPolicy{ Request: routing.ReqDefault, Response: routing.RespDefaultKeyless, }, + "aliaslist": { + Request: routing.ReqDefault, + Response: routing.RespDefaultKeyless, + Tips: map[string]string{ + routing.ReadOnlyCMD: "", + }, + }, "info": { Request: routing.ReqDefault, Response: routing.RespDefaultKeyless, @@ -156,6 +163,26 @@ var defaultPolicies = map[module]map[commandName]*routing.CommandPolicy{ }, } +// defaultPolicyKeyless reports whether name (e.g. "ft.aliaslist") is registered +// in the static policy table as a plain keyless command: default request +// routing with a keyless response policy. Commands whose slot comes from a key +// (RespDefaultHashSlot, e.g. ft.suglen) or with special request routing +// (ReqSpecial, e.g. ft.cursor) are excluded — their key position must still be +// resolved. cmdFirstKeyPosWithInfo consults this so the initial slot +// computation on a cold command-info cache matches the policy the router +// applies once the command reaches routeAndRun. +func defaultPolicyKeyless(name string) bool { + i := strings.IndexByte(name, '.') + if i < 0 { + return false + } + policy, ok := defaultPolicies[name[:i]][name[i+1:]] + if !ok { + return false + } + return policy.Request == routing.ReqDefault && policy.Response == routing.RespDefaultKeyless +} + type CommandInfoResolveFunc func(ctx context.Context, cmd Cmder) *routing.CommandPolicy type commandInfoResolver struct { diff --git a/backend/vendor/github.com/redis/go-redis/v9/commands.go b/backend/vendor/github.com/redis/go-redis/v9/commands.go index d347ffeb..c574f6e6 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/commands.go @@ -199,6 +199,9 @@ type Cmdable interface { ClientUnblock(ctx context.Context, id int64) *IntCmd ClientUnblockWithError(ctx context.Context, id int64) *IntCmd ClientMaintNotifications(ctx context.Context, enabled bool, endpointType string) *StatusCmd + ClientTracking(ctx context.Context, on bool, opt *ClientTrackingOptions) *StatusCmd + ClientTrackingOn(ctx context.Context, opt *ClientTrackingOptions) *StatusCmd + ClientTrackingOff(ctx context.Context) *StatusCmd ConfigGet(ctx context.Context, parameter string) *MapStringStringCmd ConfigResetStat(ctx context.Context) *StatusCmd ConfigSet(ctx context.Context, parameter, value string) *StatusCmd @@ -209,6 +212,7 @@ type Cmdable interface { FlushDB(ctx context.Context) *StatusCmd FlushDBAsync(ctx context.Context) *StatusCmd Info(ctx context.Context, section ...string) *StringCmd + InfoMap(ctx context.Context, section ...string) *InfoCmd LastSave(ctx context.Context) *IntCmd Save(ctx context.Context) *StatusCmd Shutdown(ctx context.Context) *StatusCmd @@ -296,8 +300,8 @@ func (c cmdable) Wait(ctx context.Context, numSlaves int, timeout time.Duration) return cmd } -func (c cmdable) WaitAOF(ctx context.Context, numLocal, numSlaves int, timeout time.Duration) *IntCmd { - cmd := NewIntCmd(ctx, "waitAOF", numLocal, numSlaves, int(timeout/time.Millisecond)) +func (c cmdable) WaitAOF(ctx context.Context, numLocal, numSlaves int, timeout time.Duration) *IntSliceCmd { + cmd := NewIntSliceCmd(ctx, "waitAOF", numLocal, numSlaves, int(timeout/time.Millisecond)) cmd.setReadTimeout(timeout) _ = c(ctx, cmd) return cmd @@ -565,6 +569,95 @@ func (c cmdable) ClientMaintNotifications(ctx context.Context, enabled bool, end return cmd } +// ClientTrackingOptions configures CLIENT TRACKING ON. See +// https://redis.io/commands/client-tracking/ for semantics. +type ClientTrackingOptions struct { + Redirect int64 + Bcast bool + Prefixes []string + OptIn bool + OptOut bool + NoLoop bool +} + +// ClientTracking enables or disables server-assisted client-side caching for +// the ONE connection that happens to serve this command. On a pooled client +// that connection is arbitrary, so this is only meaningful on a dedicated +// connection (see Client.Conn). When on is false, opt is ignored. Invalid +// option combinations are reported via the returned command's Err and nothing +// is sent to the server. +// +// Must not be combined with the built-in client-side cache: on a client +// configured with Options.ClientSideCache or ClientSideCacheConfig this +// command is rejected, because changing a pool connection's tracking state +// would silently break the cache's invalidation. +func (c cmdable) ClientTracking(ctx context.Context, on bool, opt *ClientTrackingOptions) *StatusCmd { + if !on { + return c.ClientTrackingOff(ctx) + } + return c.ClientTrackingOn(ctx, opt) +} + +// ClientTrackingOn enables tracking on the serving connection. See +// ClientTracking for the pooled-client and built-in-CSC caveats. +func (c cmdable) ClientTrackingOn(ctx context.Context, opt *ClientTrackingOptions) *StatusCmd { + args := []interface{}{"client", "tracking", "on"} + if opt != nil { + if err := validateClientTrackingOptions(opt); err != nil { + cmd := NewStatusCmd(ctx, args...) + cmd.SetErr(err) + return cmd + } + args = appendClientTrackingOptions(args, opt) + } + cmd := NewStatusCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// ClientTrackingOff disables tracking on the serving connection. See +// ClientTracking for the pooled-client and built-in-CSC caveats. +func (c cmdable) ClientTrackingOff(ctx context.Context) *StatusCmd { + cmd := NewStatusCmd(ctx, "client", "tracking", "off") + _ = c(ctx, cmd) + return cmd +} + +func validateClientTrackingOptions(opt *ClientTrackingOptions) error { + if opt.OptIn && opt.OptOut { + return errors.New("redis: CLIENT TRACKING OPTIN and OPTOUT are mutually exclusive") + } + if opt.Bcast && (opt.OptIn || opt.OptOut) { + return errors.New("redis: CLIENT TRACKING BCAST cannot be combined with OPTIN or OPTOUT") + } + if len(opt.Prefixes) > 0 && !opt.Bcast { + return errors.New("redis: CLIENT TRACKING PREFIX requires BCAST") + } + return nil +} + +func appendClientTrackingOptions(args []interface{}, opt *ClientTrackingOptions) []interface{} { + if opt.Redirect != 0 { + args = append(args, "redirect", opt.Redirect) + } + if opt.Bcast { + args = append(args, "bcast") + } + for _, p := range opt.Prefixes { + args = append(args, "prefix", p) + } + if opt.OptIn { + args = append(args, "optin") + } + if opt.OptOut { + args = append(args, "optout") + } + if opt.NoLoop { + args = append(args, "noloop") + } + return args +} + // ------------------------------------------------------------------------------------------------ func (c cmdable) ConfigGet(ctx context.Context, parameter string) *MapStringStringCmd { @@ -706,7 +799,7 @@ func (c cmdable) ReplicaOf(ctx context.Context, host, port string) *StatusCmd { } func (c cmdable) SlowLogGet(ctx context.Context, num int64) *SlowLogCmd { - cmd := NewSlowLogCmd(context.Background(), "slowlog", "get", num) + cmd := NewSlowLogCmd(ctx, "slowlog", "get", num) _ = c(ctx, cmd) return cmd } @@ -797,6 +890,11 @@ func (c *ModuleLoadexConfig) toArgs() []interface{} { // ModuleLoadex Redis `MODULE LOADEX path [CONFIG name value [CONFIG name value ...]] [ARGS args [args ...]]` command. func (c cmdable) ModuleLoadex(ctx context.Context, conf *ModuleLoadexConfig) *StringCmd { + if conf == nil { + cmd := NewStringCmd(ctx) + cmd.SetErr(errors.New("redis: ModuleLoadex nil config")) + return cmd + } cmd := NewStringCmd(ctx, conf.toArgs()...) _ = c(ctx, cmd) return cmd diff --git a/backend/vendor/github.com/redis/go-redis/v9/csc_commands.go b/backend/vendor/github.com/redis/go-redis/v9/csc_commands.go new file mode 100644 index 00000000..8c6a2b24 --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/csc_commands.go @@ -0,0 +1,257 @@ +package redis + +import ( + "bytes" + "strconv" + "strings" + + "github.com/redis/go-redis/v9/internal/proto" +) + +// defaultCacheableCommands is the allow-list of read-only, deterministic +// commands whose responses may be stored in the client-side cache. Keys are +// lowercase to match baseCmd.Name() on the hot path. +var defaultCacheableCommands = map[string]struct{}{ + // String commands + "get": {}, "mget": {}, "getbit": {}, "getrange": {}, + "strlen": {}, "substr": {}, + // Hash commands + "hget": {}, "hgetall": {}, "hmget": {}, + "hkeys": {}, "hvals": {}, "hlen": {}, + "hexists": {}, "hstrlen": {}, + // List commands + "lindex": {}, "llen": {}, "lpos": {}, "lrange": {}, + // Set commands + "scard": {}, "sismember": {}, "smembers": {}, "smismember": {}, + "sdiff": {}, "sinter": {}, "sintercard": {}, "sunion": {}, + // Sorted-set commands + "zcard": {}, "zcount": {}, "zlexcount": {}, "zmscore": {}, + "zrange": {}, "zrangebylex": {}, "zrangebyscore": {}, + "zrank": {}, "zrevrange": {}, "zrevrangebylex": {}, + "zrevrangebyscore": {}, "zrevrank": {}, "zscore": {}, + "zdiff": {}, "zinter": {}, "zunion": {}, + // Bit commands + "bitcount": {}, "bitfield_ro": {}, "bitpos": {}, + // Key/generic commands + "exists": {}, "type": {}, "sort_ro": {}, "lcs": {}, + // Geo commands + "geodist": {}, "geohash": {}, "geopos": {}, "geosearch": {}, + "georadiusbymember_ro": {}, "georadius_ro": {}, + // Stream commands. XREAD is deliberately excluded: it supports BLOCK, and + // its $/+ IDs are state-relative, so identical args are not deterministic. + // XPENDING is excluded for the same class of reason: its extended form + // returns wall-clock-relative idle times and its IDLE filter is + // time-dependent, so identical args yield different correct results with + // no key modification (and therefore no invalidation). + "xlen": {}, "xrange": {}, "xrevrange": {}, + // JSON (RedisJSON) commands + "json.get": {}, "json.mget": {}, "json.arrindex": {}, "json.arrlen": {}, + "json.objkeys": {}, "json.objlen": {}, "json.resp": {}, + "json.strlen": {}, "json.type": {}, + // TimeSeries commands + "ts.get": {}, "ts.info": {}, "ts.range": {}, "ts.revrange": {}, +} + +// isCacheable reports whether cmd is eligible for client-side caching: its +// name is on the allow-list and it operates on at least one key. +func isCacheable(cmd Cmder) bool { + // Commands such as RawWriteToCmd stream replies directly to an io.Writer. + // Capturing their replies for CSC would buffer the entire response first, + // defeating their streaming and allocation guarantees. + if cmd.NoRetry() { + return false + } + if _, ok := defaultCacheableCommands[cmd.Name()]; !ok { + return false + } + // SORT_RO ... BY/GET reads pattern keys that extractRedisKeys can't + // enumerate, so its invalidations would be dropped and the result go stale. + // Plain SORT_RO is fine. + if cmd.Name() == "sort_ro" && sortROHasByGet(cmd) { + return false + } + return cmdFirstKeyPosWithInfo(cmd, nil) != 0 +} + +// sortROHasByGet reports whether a SORT_RO invocation uses BY or GET +// (case-insensitive), scanning past the command name and key. stringArg +// normalizes string, *string, and []byte tokens. +func sortROHasByGet(cmd Cmder) bool { + for i := 2; i < len(cmd.Args()); i++ { + if s := cmd.stringArg(i); strings.EqualFold(s, "by") || strings.EqualFold(s, "get") { + return true + } + } + return false +} + +// isClientTrackingCmd reports whether cmd is a CLIENT TRACKING subcommand (any +// mode: ON, OFF, or with options). Name and stringArg normalize string, +// *string, and []byte arguments. +func isClientTrackingCmd(cmd Cmder) bool { + return cmd.Name() == "client" && strings.EqualFold(cmd.stringArg(1), "tracking") +} + +// isSelectCmd reports whether cmd changes the selected database on its +// connection. CSC keys are namespaced with Options.DB, so a runtime SELECT +// would make the connection's actual database diverge from the cache namespace. +func isSelectCmd(cmd Cmder) bool { + return cmd.Name() == "select" +} + +// isAuthCmd reports whether cmd changes the authenticated user on its +// connection. The cache namespace is fixed from Options.Username, so runtime +// authentication would make the connection identity diverge from it. +func isAuthCmd(cmd Cmder) bool { + return cmd.Name() == "auth" +} + +// isProtocolChangingHelloCmd reports whether HELLO includes a protocol version +// (and can therefore switch a tracked RESP3 connection to RESP2). A bare HELLO +// only reports connection properties and is safe. +func isProtocolChangingHelloCmd(cmd Cmder) bool { + return cmd.Name() == "hello" && len(cmd.Args()) > 1 +} + +// isResetCmd reports whether cmd resets all server-side connection state. +// RESET disables tracking, switches to RESP2, deauthenticates, and changes +// other state that a pooled CSC connection relies on. +func isResetCmd(cmd Cmder) bool { + return cmd.Name() == "reset" +} + +// isSubscribeCmd reports whether a raw command would turn an ordinary pooled +// connection into a Pub/Sub connection. Pub/Sub pushes are deliberately left +// for the dedicated PubSub reader, so the CSC drainer cannot safely own such a +// connection. +func isSubscribeCmd(cmd Cmder) bool { + switch cmd.Name() { + case "subscribe", "psubscribe", "ssubscribe": + return true + default: + return false + } +} + +// buildCacheKey returns the RESP-encoded form of the command's argument list, +// used as a collision-free canonical cache key. ok is false when the writer +// cannot marshal the arguments, in which case the caller must skip caching +// rather than bucket the command under an empty key. +func buildCacheKey(cmd Cmder) (string, bool) { + args := cmd.Args() + if len(args) == 0 { + return "", false + } + var buf bytes.Buffer + if err := proto.NewWriter(&buf).WriteArgs(args); err != nil { + return "", false + } + return buf.String(), true +} + +// keyArg renders the key argument at pos exactly as proto.Writer sends it to +// the server, so invalidation lookups match the key names in the server's +// "invalidate" pushes. Only types whose stringArg rendering is byte-identical +// to the wire encoding are accepted (fmt.Sprint of any integer matches the +// writer's base-10 strconv output); for anything else — pointers, bools, +// times, durations, floats, BinaryMarshaler values — the rendering can +// diverge, the invalidation would never match, and the entry would be served +// stale forever, so ok=false and the caller skips caching (see processCached). +func keyArg(cmd Cmder, pos int) (string, bool) { + args := cmd.Args() + if pos < 0 || pos >= len(args) { + return "", false + } + switch args[pos].(type) { + case string, []byte, + int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64: + return cmd.stringArg(pos), true + } + return "", false +} + +// extractRedisKeys returns the Redis key arguments from cmd. The result lets +// the cache map incoming invalidations back to affected entries. Returns nil +// (caller skips caching) when any key +// argument cannot be rendered in its wire form (see keyArg). +func extractRedisKeys(cmd Cmder) []string { + firstKey := cmdFirstKeyPosWithInfo(cmd, nil) + if firstKey == 0 { + return nil + } + + argsLen := len(cmd.Args()) + if firstKey >= argsLen { + return nil + } + + switch cmd.Name() { + // All remaining args from firstKeyPos are keys. + case "mget", "exists", "sdiff", "sinter", "sunion": + keys := make([]string, 0, argsLen-firstKey) + for i := firstKey; i < argsLen; i++ { + k, ok := keyArg(cmd, i) + if !ok { + return nil + } + keys = append(keys, k) + } + return keys + + // Numkeys pattern: numkeys at args[1], keys from args[2]. + case "sintercard", "zdiff", "zinter", "zunion": + if argsLen < 3 { + return nil + } + numKeys, err := strconv.Atoi(cmd.stringArg(1)) + if err != nil || numKeys <= 0 { + return nil + } + keys := make([]string, 0, numKeys) + for i := 2; i < 2+numKeys && i < argsLen; i++ { + k, ok := keyArg(cmd, i) + if !ok { + return nil + } + keys = append(keys, k) + } + return keys + + // LCS: exactly two consecutive keys starting at firstKeyPos. + case "lcs": + if firstKey+1 >= argsLen { + return nil + } + k1, ok1 := keyArg(cmd, firstKey) + k2, ok2 := keyArg(cmd, firstKey+1) + if !ok1 || !ok2 { + return nil + } + return []string{k1, k2} + + // JSON.MGET: keys from firstKeyPos to second-to-last (last arg is the + // JSON path, not a key). + case "json.mget": + lastKey := argsLen - 2 + if lastKey < firstKey { + return nil + } + keys := make([]string, 0, lastKey-firstKey+1) + for i := firstKey; i <= lastKey; i++ { + k, ok := keyArg(cmd, i) + if !ok { + return nil + } + keys = append(keys, k) + } + return keys + } + + // Single key at firstKeyPos (GET, HGET, LRANGE, ...). + k, ok := keyArg(cmd, firstKey) + if !ok { + return nil + } + return []string{k} +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/csc_integration.go b/backend/vendor/github.com/redis/go-redis/v9/csc_integration.go new file mode 100644 index 00000000..f9b41352 --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/csc_integration.go @@ -0,0 +1,922 @@ +package redis + +import ( + "bytes" + "context" + "errors" + "reflect" + "runtime" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/redis/go-redis/v9/internal" + "github.com/redis/go-redis/v9/internal/pool" + "github.com/redis/go-redis/v9/internal/proto" + "github.com/redis/go-redis/v9/push" +) + +// cscRegisterCleanups arranges for a client dropped without Close to stop its +// background CSC drainer. The drainer's exit path revokes its pool's cache +// coverage; the runtime cleanup itself stays non-blocking and never captures +// *Client, so the wrapper remains collectible. +func cscRegisterCleanups(c *Client) { + h := c.baseClient.cscDrainHandle + if h == nil { + return + } + // Capture cscActive (a standalone *atomic.Bool, not *Client) so the cleanup + // also stops clones from serving once the drainer is gone. + active := c.baseClient.cscActive + runtime.AddCleanup(c, func(h *cscDrainHandle) { + if active != nil { + active.Store(false) + } + h.signalStop() + }, h) +} + +// ClientSideCacheConfig configures the built-in client-side cache. Pass a +// non-nil value to Options.ClientSideCacheConfig to enable caching on a RESP3 +// client. +// +// Experimental: this API may change in a minor release. +type ClientSideCacheConfig = CacheConfig + +const ( + invalidatePushName = "invalidate" + // cscNamespaceSep separates fixed-width/logically-delimited namespace parts + // from the command or Redis key. + cscNamespaceSep = "\x00" +) + +// cscNamespacePrefix scopes a shared cache by database and fixed ACL identity. +// Password rotation does not change identity; provider-backed identities are +// rejected before attachment. +func cscNamespacePrefix(db int, username string) string { + return strconv.Itoa(db) + cscNamespaceSep + + strconv.Itoa(len(username)) + ":" + username + cscNamespaceSep +} + +func cscNamespacedKey(prefix, key string) string { + return prefix + key +} + +// invalidateHandler propagates RESP3 "invalidate" push notifications into the +// shared client-side cache. keyPrefix scopes incoming key names so a shared +// cache cannot collide across databases or fixed ACL identities. +// +// The binding (cache, keyPrefix) is mutable under mu: the owning client's teardown +// RELEASES it (cache=nil) instead of unregistering the handler, so the handler +// can stay registered protected — application code holding the processor +// cannot silently unregister invalidation out from under a live client — while +// a successor client on the same processor can still rebind it (see +// registerInvalidateHandler). +type invalidateHandler struct { + mu sync.RWMutex + cache Cache + keyPrefix string + users int +} + +// HandlePushNotification decodes ["invalidate", ] notifications. A nil +// payload is emitted on FLUSHDB/FLUSHALL and triggers a full cache flush. +func (h *invalidateHandler) HandlePushNotification( + _ context.Context, _ push.NotificationHandlerContext, notification []interface{}, +) error { + h.mu.RLock() + cache, keyPrefix := h.cache, h.keyPrefix + h.mu.RUnlock() + if cache == nil || len(notification) < 2 { + return nil + } + + switch payload := notification[1].(type) { + case nil: + cache.Flush() + case []interface{}: + for _, k := range payload { + var name string + switch v := k.(type) { + case string: + name = v + case []byte: + name = string(v) + default: + continue + } + cache.DeleteByRedisKey(cscNamespacedKey(keyPrefix, name)) + } + } + return nil +} + +func (h *invalidateHandler) release() { + h.mu.Lock() + if h.users > 0 { + h.releaseLocked() + } + h.mu.Unlock() +} + +func (h *invalidateHandler) releaseLocked() { + h.users-- + if h.users == 0 { + h.cache = nil + h.keyPrefix = "" + } +} + +// sameCache compares Cache interface values without panicking when an +// implementation uses a non-comparable value type. +func sameCache(a, b Cache) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + typ := reflect.TypeOf(a) + return typ == reflect.TypeOf(b) && typ.Comparable() && a == b +} + +func isNilCache(cache Cache) bool { + if cache == nil { + return true + } + v := reflect.ValueOf(cache) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + default: + return false + } +} + +// errInvalidateHandlerBound: piggybacking on a handler bound to a live +// different cache would leave the new cache uninvalidated. +var errInvalidateHandlerBound = errors.New(`csc: a different "invalidate" push handler is already registered`) + +// bindTo binds the handler to (cache, keyPrefix). Success when that is already the +// binding (a derived Client.Conn sharing the parent's processor and cache) or +// when the handler was released by a previous owner's teardown (rebind); +// errInvalidateHandlerBound otherwise. +func (h *invalidateHandler) bindTo(cache Cache, keyPrefix string) error { + h.mu.Lock() + defer h.mu.Unlock() + switch { + case sameCache(h.cache, cache) && h.keyPrefix == keyPrefix: + h.users++ + return nil + case h.cache == nil: + h.cache, h.keyPrefix = cache, keyPrefix + h.users = 1 + return nil + default: + return errInvalidateHandlerBound + } +} + +// lookupInvalidateHandler returns the processor's CSC invalidate handler, nil +// when absent or foreign. +func lookupInvalidateHandler(p push.NotificationProcessor) *invalidateHandler { + if p == nil { + return nil + } + h, _ := p.GetHandler(invalidatePushName).(*invalidateHandler) + return h +} + +func registerInvalidateHandler(p push.NotificationProcessor, cache Cache, keyPrefix string) error { + if p == nil || cache == nil { + return nil + } + if existing := p.GetHandler(invalidatePushName); existing != nil { + h, ok := existing.(*invalidateHandler) + if !ok { + return errInvalidateHandlerBound + } + return h.bindTo(cache, keyPrefix) + } + // VoidProcessor (RESP2) returns an error here; the caller treats it as + // "CSC not available" rather than fatal. Registered PROTECTED: application + // code holding the processor must not be able to unregister invalidation + // under a live client (that would serve unbounded-stale hits with no + // signal); owner teardown releases the BINDING instead of the handler. + err := p.RegisterHandler(invalidatePushName, &invalidateHandler{ + cache: cache, + keyPrefix: keyPrefix, + users: 1, + }, true) + if err == nil { + return nil + } + // Another client can register the same protected handler between GetHandler + // and RegisterHandler. Re-read it and accept the compatible binding. + if existing := p.GetHandler(invalidatePushName); existing != nil { + h, ok := existing.(*invalidateHandler) + if !ok { + return errInvalidateHandlerBound + } + return h.bindTo(cache, keyPrefix) + } + return err +} + +// attachCSC dispatches to the invalidation strategy in +// Options.ClientSideCacheStrategy. Safe with a nil cache; on failure c.csc stays +// nil and commands fall back to normal round-trips. Adding a strategy: a new +// CSCStrategy constant plus cases in Options.init and here. +func (c *baseClient) attachCSC(ctx context.Context, cache Cache) { + if isNilCache(cache) || c.opt.Protocol != 3 { + return + } + // Credential providers may return a different ACL identity over the + // client's lifetime (or per context/connection), while the cache namespace + // is fixed when the client is created. Fixed credentials remain safe because + // the ACL username is included in the length-delimited namespace below. + if c.opt.StreamingCredentialsProvider != nil || + c.opt.CredentialsProviderContext != nil || + c.opt.CredentialsProvider != nil { + internal.Logger.Printf(ctx, + "redis: client-side caching is disabled with credential providers") + return + } + c.cscKeyPrefix = cscNamespacePrefix(c.opt.DB, c.opt.Username) + switch c.opt.ClientSideCacheStrategy { + case CSCStrategySharedTracking: + c.attachSharedTrackingCSC(ctx, cache) + default: + // Options.init clamps unknown strategies to SharedTracking; delegate anyway. + c.attachSharedTrackingCSC(ctx, cache) + } +} + +// attachSharedTrackingCSC wires SharedTracking: one shared cache, per-conn CLIENT +// TRACKING, a background drainer, and the owning-conn eviction hook. DB-0 only: +// tracking is bound to the conn's DB and a runtime SELECT does not re-key it. +func (c *baseClient) attachSharedTrackingCSC(ctx context.Context, cache Cache) { + if c.opt.DB != 0 { + internal.Logger.Printf(ctx, + "csc: client-side caching is restricted to DB 0; disabling CSC for client configured with DB=%d. "+ + "Use one client per DB if you need caching against non-zero databases.", c.opt.DB) + return + } + // A pooler without idle-conn draining (e.g. Client.Conn's StickyConnPool) + // can't apply buffered invalidations, so stay uncached. + if _, ok := c.connPool.(idleConnDrainer); !ok { + return + } + // The lifecycle hook serializes cache publication with connection removal + // and socket replacement. Without it, a reply can become visible after its + // tracking coverage is gone. + reg, ok := c.connPool.(poolHookSupport) + if !ok || !reg.SupportsPoolHooks() { + return + } + if err := registerInvalidateHandler(c.pushProcessor, cache, c.cscKeyPrefix); err != nil { + internal.Logger.Printf(ctx, "csc: failed to register invalidate handler: %v", err) + return + } + c.csc = cache + c.registerConnEvictHook(cache, reg) + c.startBackgroundDrainer() +} + +// cscHook returns the shared evict-on-remove hook, nil when CSC is off. +func (c *baseClient) cscHook() *cscEvictOnRemoveHook { + h, _ := c.cscPoolHook.(*cscEvictOnRemoveHook) + return h +} + +// cscInstallConnCloseHook evicts cn's owned entries on any close — including the +// ConnMaxLifetime/idle retirement path (CloseConn) that bypasses the OnRemove +// hook — so entries don't outlive the server tracking dropped at close. Uses the +// onCscClose slot so it doesn't clobber streaming-credentials cleanup. +func (c *baseClient) cscInstallConnCloseHook(cn *pool.Conn) { + cn.SetOnCscClose(func() error { + c.cscOnConnClose(cn.GetID()) + return nil + }) +} + +// cscInstallConnReinitHook invalidates the old socket's cache coverage before +// SetNetConnAndInitConn replaces it. The later init can then safely enable +// tracking for the new socket without a post-swap publication window. +func (c *baseClient) cscInstallConnReinitHook(cn *pool.Conn) { + cn.SetOnCscReinit(func() { + c.cscEvictOwnedEntries(cn.GetID()) + }) +} + +// cscOnConnClose evicts a closing conn's entries: via the shared hook (which +// records the removed-ring, closing the close-before-fulfill race), else scoped +// EvictByConn on the owning cache. +func (c *baseClient) cscOnConnClose(connID uint64) { + if h := c.cscHook(); h != nil { + h.markRemoved(connID) + return + } + if c.csc != nil { + c.csc.EvictByConn(connID) + } +} + +// poolHookSupport is the pool capability SharedTracking needs to serialize +// cache publication with connection removal and reinitialization. +type poolHookSupport interface { + AddPoolHook(hook pool.PoolHook) + RemovePoolHook(hook pool.PoolHook) + SupportsPoolHooks() bool +} + +// cscEvictOnRemoveHook evicts a connection's owned entries when the pool removes +// it (the server stops delivering their invalidations — Window 2), and tracks +// per-conn init generations so fulfillCached can catch a value whose owning +// conn was removed or re-initialized mid-fetch. +type cscEvictOnRemoveHook struct { + evictor Cache + + mu sync.Mutex + // initGen counts a live conn's socket (re)initializations: bumped by + // cscEvictOwnedEntries before its eviction (first init included, so every + // serving conn has gen >= 1), deleted on removal/close. fulfillCached + // compares it with the generation captured at reply time. + initGen map[uint64]uint64 +} + +func (h *cscEvictOnRemoveHook) OnGet(_ context.Context, _ *pool.Conn, _ bool) (bool, error) { + return true, nil +} + +func (h *cscEvictOnRemoveHook) OnPut(_ context.Context, _ *pool.Conn) (shouldPool, shouldRemove bool, err error) { + return true, false, nil +} + +func (h *cscEvictOnRemoveHook) OnRemove(_ context.Context, cn *pool.Conn, _ error) { + if cn == nil { + return + } + h.markRemoved(cn.GetID()) +} + +// markRemoved forgets connID's generation, then evicts. Forgetting before +// evicting lets a racing fulfillCached see the change (a served conn's captured +// generation is >= 1, an absent entry reads 0) and drop an entry created after +// the eviction — closing the close-before-fulfill race. +func (h *cscEvictOnRemoveHook) markRemoved(connID uint64) { + h.forgetConn(connID) + h.evictor.EvictByConn(connID) +} + +// bumpInitGen advances connID's coverage generation. On reinit it is called by +// the pre-swap hook, before the old socket and its server-side tracking table +// are replaced. +func (h *cscEvictOnRemoveHook) bumpInitGen(connID uint64) { + h.mu.Lock() + if h.initGen == nil { + h.initGen = make(map[uint64]uint64) + } + h.initGen[connID]++ + h.mu.Unlock() +} + +// invalidateConnCoverage revokes all cache coverage associated with connID. +// Bumping before eviction also rejects an in-flight fetch that completed on the +// connection just before it left the parent's invalidation drainer. +func (h *cscEvictOnRemoveHook) invalidateConnCoverage(connID uint64) { + h.bumpInitGen(connID) + h.evictor.EvictByConn(connID) +} + +// initGenOf returns connID's current init generation (0 if never bumped). +func (h *cscEvictOnRemoveHook) initGenOf(connID uint64) uint64 { + h.mu.Lock() + defer h.mu.Unlock() + return h.initGen[connID] +} + +// forgetConn drops connID's init-generation entry: the conn was removed/closed, +// or its init failed before ever serving (the pubsub path would otherwise leak +// the entry — no OnRemove hook, close hook not yet installed). +func (h *cscEvictOnRemoveHook) forgetConn(connID uint64) { + h.mu.Lock() + delete(h.initGen, connID) + h.mu.Unlock() +} + +// fulfillOwnedIfCovered linearizes the final coverage check with connection +// removal/re-init generation changes. Holding h.mu through FulfillOwned means +// either the old generation is rejected before the placeholder becomes valid, +// or publication wins first and the subsequent lifecycle path evicts it before +// closing/replacing the tracked socket. +func (h *cscEvictOnRemoveHook) fulfillOwnedIfCovered( + cacheKey string, + token, ownerConnID, capturedGen uint64, + value []byte, +) bool { + h.mu.Lock() + defer h.mu.Unlock() + if h.initGen[ownerConnID] != capturedGen { + return false + } + return h.evictor.FulfillOwned(cacheKey, token, ownerConnID, value) +} + +// invalidateAllCoverage revokes every connection generation known to this +// client's pool and evicts the entries those connections own. Incrementing +// instead of deleting keeps in-flight fetches that captured an old generation +// from publishing after a drainer stops. +func (h *cscEvictOnRemoveHook) invalidateAllCoverage() { + h.mu.Lock() + connIDs := make([]uint64, 0, len(h.initGen)) + for connID := range h.initGen { + h.initGen[connID]++ + connIDs = append(connIDs, connID) + } + h.mu.Unlock() + + for _, connID := range connIDs { + h.evictor.EvictByConn(connID) + } +} + +// registerConnEvictHook wires the required OnRemove eviction hook. +func (c *baseClient) registerConnEvictHook(cache Cache, reg poolHookSupport) { + h := &cscEvictOnRemoveHook{evictor: cache, initGen: make(map[uint64]uint64)} + reg.AddPoolHook(h) + c.cscPoolHook = h +} + +// cscEvictOwnedEntries evicts connID's entries on first init or immediately +// before a reinit/handoff replaces the socket and its tracking table. It +// prefers the shared hook (so Conn/Tx, which carry it but have a nil csc, still +// evict from the parent cache). Scoped only — no removed-ring (the conn keeps +// serving, and the ring never ages out); the fulfill-vs-re-init race is closed +// by the init-generation bump instead. No custom-cache flush (this also runs on +// first init). +func (c *baseClient) cscEvictOwnedEntries(connID uint64) { + if h := c.cscHook(); h != nil { + h.invalidateConnCoverage(connID) + return + } + if c.csc == nil { + return + } + c.csc.EvictByConn(connID) +} + +// newStickyConnPool creates a derived sticky pool and revokes the claimed +// connection's parent-cache ownership before it becomes unreachable to the +// parent's idle-connection drainer. +func (c *baseClient) newStickyConnPool() *pool.StickyConnPool { + sticky := pool.NewStickyConnPool(c.connPool) + if h := c.cscHook(); h != nil { + sticky.SetOnFirstConn(func(cn *pool.Conn) { + if cn != nil { + h.invalidateConnCoverage(cn.GetID()) + } + }) + } + return sticky +} + +// cscFetchCapture receives, from the successful attempt's reply read — while +// the serving connection is still held — everything the CSC fetch path needs to +// attribute the cached entry: the raw RESP reply, the conn id, and the conn's +// CSC init generation. The generation must be captured before the conn is +// released: a handoff queued at Put can re-init the socket (bumping the +// generation) before fulfillCached runs. +type cscFetchCapture struct { + raw []byte + connID uint64 + initGen uint64 +} + +// cscConnInitGen returns connID's CSC init generation, captured by _process at +// reply time (while the conn is still held) and compared by fulfillCached via +// fulfillOwnedIfCovered. Zero without an active evict-on-remove hook. +func (c *baseClient) cscConnInitGen(connID uint64) uint64 { + if h := c.cscHook(); h != nil { + return h.initGenOf(connID) + } + return 0 +} + +// cscForgetConn drops connID's init-generation entry when initialization does +// not establish tracked coverage, either because init failed or tracking was +// rejected and CSC was disabled. +func (c *baseClient) cscForgetConn(connID uint64) { + if h := c.cscHook(); h != nil { + h.forgetConn(connID) + } +} + +// errClientTrackingWithCSC rejects CLIENT TRACKING on clients with built-in CSC +// (see the guards in baseClient.process and generalProcessPipeline). The raw +// escape hatches — Do(ctx, "client", "tracking", ...) with string or []byte +// args, and pipelines — are also caught: the guard matches on the command's +// leading args, not the typed method. +var errClientTrackingWithCSC = errors.New( + "redis: CLIENT TRACKING is not allowed when client-side caching is enabled") + +// errSelectWithCSC rejects runtime SELECT on clients with built-in CSC. Cache +// keys use Options.DB, while SELECT mutates only the chosen pool connection. +var errSelectWithCSC = errors.New( + "redis: SELECT is not allowed when client-side caching is enabled") + +// errAuthWithCSC rejects runtime authentication because it can change one +// connection's ACL identity without changing the client's fixed cache namespace. +var errAuthWithCSC = errors.New( + "redis: AUTH is not allowed when client-side caching is enabled") + +// errHelloWithCSC rejects HELLO with arguments because it can switch a tracked +// connection out of RESP3 (and can also change authentication). +var errHelloWithCSC = errors.New( + "redis: HELLO with arguments is not allowed when client-side caching is enabled") + +// errResetWithCSC rejects RESET because it disables tracking and switches the +// connection to RESP2. +var errResetWithCSC = errors.New( + "redis: RESET is not allowed when client-side caching is enabled") + +// errSubscribeWithCSC rejects raw subscriptions on the ordinary pool. The +// typed Subscribe methods use dedicated PubSub connections and remain allowed. +var errSubscribeWithCSC = errors.New( + "redis: SUBSCRIBE is not allowed on pooled connections when client-side caching is enabled") + +// cscCommandError rejects commands that can make a pooled connection's state +// diverge from the assumptions used by CSC. +func (c *baseClient) cscCommandError(cmd Cmder) error { + // The successful attachment signal is shared with derived clients. + // initConn's internal command wrapper is exempt during library setup. + if !c.cscTrackingRequested() || c.allowClientTracking { + return nil + } + switch { + case isClientTrackingCmd(cmd): + return errClientTrackingWithCSC + case isSelectCmd(cmd): + return errSelectWithCSC + case isAuthCmd(cmd): + return errAuthWithCSC + case isProtocolChangingHelloCmd(cmd): + return errHelloWithCSC + case isResetCmd(cmd): + return errResetWithCSC + case isSubscribeCmd(cmd): + return errSubscribeWithCSC + default: + return nil + } +} + +// cscDrainHandle owns the drainer lifecycle and serializes client teardown. +// stop signals shutdown; done is closed on exit so Close can join. +type cscDrainHandle struct { + stop chan struct{} + done chan struct{} + stopOnce sync.Once + teardownOnce sync.Once + handlerCloseOnce sync.Once + closeOnce sync.Once + closeErr error + invalidateHandler *invalidateHandler +} + +// signalStop closes stop at most once (so Close and the AddCleanup safety net +// can't double-close) and does not join — a GC cleanup must not block. +func (h *cscDrainHandle) signalStop() { + h.stopOnce.Do(func() { close(h.stop) }) +} + +// cscHandlerClient is exposed only through the background drainer's handler +// context. Close must return before the handler does, otherwise it would wait +// for the drainer goroutine that is currently invoking the handler. +type cscHandlerClient struct { + *baseClient +} + +func (c cscHandlerClient) Close() error { + h := c.cscDrainHandle + if h == nil { + return c.baseClient.Close() + } + h.handlerCloseOnce.Do(func() { + // Close has logically started: stop cache hits immediately and let the + // drainer exit as soon as this handler returns. + if c.cscActive != nil { + c.cscActive.Store(false) + } + h.signalStop() + go func() { + if err := c.baseClient.Close(); err != nil { + internal.Logger.Printf(context.Background(), "csc: deferred client close failed: %v", err) + } + }() + }) + return nil +} + +// cscMinDrainInterval floors a user-supplied DrainInterval: sub-millisecond +// timers are unreliable (https://github.com/golang/go/issues/53824). +const cscMinDrainInterval = time.Millisecond + +// cscDrainInterval returns DrainInterval clamped to cscMinDrainInterval, or the +// default (cscDrainSkipWindow) when unset. +func (c *baseClient) cscDrainInterval() time.Duration { + if cfg := c.opt.ClientSideCacheConfig; cfg != nil && cfg.DrainInterval > 0 { + if cfg.DrainInterval < cscMinDrainInterval { + return cscMinDrainInterval + } + return cfg.DrainInterval + } + return cscDrainSkipWindow +} + +// idleConnDrainer is the pooler capability the drainer needs (*pool.ConnPool has +// it). attachSharedTrackingCSC leaves a pooler without it uncached, rather than +// serve entries nothing would invalidate. +type idleConnDrainer interface { + DrainIdleConns(ctx context.Context, st *pool.DrainState, fn func(cn *pool.Conn) error) +} + +// startBackgroundDrainer launches the per-client invalidation drainer: each tick +// runs one pool.DrainIdleConns pass, draining idle conns' buffered push frames. +// No-op for poolers that don't implement idleConnDrainer. +func (c *baseClient) startBackgroundDrainer() { + cp, ok := c.connPool.(idleConnDrainer) + if !ok { + return + } + if c.cscDrainHandle != nil { + return // already running (startBackgroundDrainer runs once, in NewClient) + } + h := &cscDrainHandle{ + stop: make(chan struct{}), + done: make(chan struct{}), + invalidateHandler: lookupInvalidateHandler(c.pushProcessor), + } + c.cscDrainHandle = h + active := &atomic.Bool{} + active.Store(true) + c.cscActive = active + interval := c.cscDrainInterval() + // Custom-processor drain errors are connection-fatal (drainPushNotifications), + // so a PERSISTENTLY failing custom processor would turn every tick into a + // conn removal + redial — a sustained dial storm. Damping: after + // cscDrainCustomErrCap consecutive fatal custom-processor drains, disable + // CSC serving and stop the drainer (with one log line) instead of churning. + // Built-in processor errors are real conn desyncs and are never damped. + _, builtinProc := c.pushProcessor.(*push.Processor) + go func() { + defer func() { + active.Store(false) + if c.cscPoolHook != nil { + if reg, ok := c.connPool.(poolHookSupport); ok { + reg.RemovePoolHook(c.cscPoolHook) + } + } + if hook := c.cscHook(); hook != nil { + hook.invalidateAllCoverage() + } + if h.invalidateHandler != nil { + h.invalidateHandler.release() + } + close(h.done) + }() + ticker := time.NewTicker(interval) + defer ticker.Stop() + // st persists round/visited across ticks; single-goroutine, no lock. + var st pool.DrainState + consecFatal := 0 + drain := func(cn *pool.Conn) error { + processorSucceeded, err := c.drainPushNotifications(cn) + switch { + case err != nil: + consecFatal++ + case processorSucceeded: + // A successful processor invocation resets consecutive + // failures. A conn skipped without invoking the processor — + // including a clean replacement after a fatal drain — does + // not reset the counter. + consecFatal = 0 + } + return err + } + for { + select { + case <-h.stop: + return + case <-ticker.C: + if !active.Load() { + return + } + // ctx bounds the whole pass; the drain read has its own hard deadline. + cycleCtx, cancel := context.WithTimeout(context.Background(), interval/2) + cp.DrainIdleConns(cycleCtx, &st, drain) + cancel() + if !builtinProc && consecFatal >= cscDrainCustomErrCap { + internal.Logger.Printf(context.Background(), + "csc: disabling client-side caching: the custom push notification processor failed %d consecutive drains "+ + "(each failure removes a connection because the reader may be mid-frame); "+ + "caching cannot be kept fresh safely with this processor", consecFatal) + return + } + } + } + }() +} + +// disableCSCServing atomically stops cache hits and revokes all tracked +// connection coverage. The owner drainer observes the shared active flag on its +// next tick, including when a derived Conn or Tx discovered the incompatibility. +func (c *baseClient) disableCSCServing(ctx context.Context, reason string) { + active := c.cscActive + if active == nil || !active.CompareAndSwap(true, false) { + return + } + if hook := c.cscHook(); hook != nil { + hook.invalidateAllCoverage() + } + internal.Logger.Printf(ctx, "csc: disabling client-side caching: %s", reason) +} + +// stopBackgroundDrainer joins the drainer goroutine and flushes an owned cache. +// The drainer's exit path releases its handler binding and pool hook, including +// when it stops itself. Owner-only: clones have no handle and return early. +// The fields are never cleared here — fulfillCached reads cscPoolHook on the hot +// path, so niling under a concurrent Close would race; teardownOnce makes repeat +// Close idempotent instead. +func (c *baseClient) stopBackgroundDrainer() { + h := c.cscDrainHandle + if h == nil { + return + } + h.teardownOnce.Do(func() { + // Stop serving cache hits on any clone before the drainer is gone. + if c.cscActive != nil { + c.cscActive.Store(false) + } + h.signalStop() + <-h.done + // The drainer's exit defer revoked and evicted this pool's coverage + // before closing done, including for injected caches shared elsewhere. + if c.cscOwnsCache && c.csc != nil { + c.csc.Flush() + } + }) +} + +// applyCachedReply populates cmd from a previously captured raw RESP reply by +// replaying it through the command's own readReply. +func applyCachedReply(cmd Cmder, raw []byte) error { + return cmd.readReply(proto.NewReaderSize(bytes.NewReader(raw), len(raw)+1)) +} + +// isCacheableReplyResult reports whether a fully read Redis reply can be +// cached. redis.Nil is a normal negative lookup, not a transport/protocol +// failure; tracking will invalidate it if the key is later created. +func isCacheableReplyResult(err error) bool { + return err == nil || err == Nil +} + +// cscDrainSkipWindow is the default SharedTracking drain period (overridable via +// ClientSideCacheConfig.DrainInterval). A buffered invalidation is picked up within +// roughly one round; MaxStaleness, when configured, is the hard time-based backstop. +const cscDrainSkipWindow = 5 * time.Millisecond + +// cscDrainHardReadCap is the hard socket read deadline the drainer applies via +// Conn.WithReaderHardDeadline. It bounds only a rare partial-frame mid-read. A +// var (not const) so the tuning harness can sweep it. +var cscDrainHardReadCap = 50 * time.Millisecond + +// cscDrainProbeReadCap bounds the non-consuming one-byte probe used only when +// an opaque transport may hold data that the socket readiness check cannot see. +const cscDrainProbeReadCap = 50 * time.Microsecond + +// cscDrainCustomErrCap is the number of CONSECUTIVE fatal custom-processor +// drain errors after which the drainer disables CSC instead of removing (and +// redialing) a connection per tick indefinitely. +const cscDrainCustomErrCap = 8 + +// processCached runs the Get-Reserve-Fulfill lifecycle for a cacheable command. +// Only invoked after process has verified that CSC is active and cmd is +// eligible. +func (c *baseClient) processCached(ctx context.Context, cmd Cmder, state *processState) error { + if err := ctx.Err(); err != nil { + return err + } + + // Once the drainer has stopped (owner Close, or the owner dropped without + // Close), no invalidations flow — a surviving clone must not serve stale hits. + if a := c.cscActive; a != nil && !a.Load() { + return c.processWithRetry(ctx, cmd, nil, state) + } + + rawKey, ok := buildCacheKey(cmd) + if !ok { + return c.processWithRetry(ctx, cmd, nil, state) + } + + redisKeys := extractRedisKeys(cmd) + if len(redisKeys) == 0 { + // Without a key list we cannot react to invalidations for this command. + return c.processWithRetry(ctx, cmd, nil, state) + } + + keyPrefix := c.cscKeyPrefix + if keyPrefix == "" { + // A successfully attached client always has a namespace. Fail closed if + // an incomplete custom baseClient reaches this path. + return c.processWithRetry(ctx, cmd, nil, state) + } + key := cscNamespacedKey(keyPrefix, rawKey) + nsRedisKeys := make([]string, len(redisKeys)) + for i, k := range redisKeys { + nsRedisKeys[i] = cscNamespacedKey(keyPrefix, k) + } + + // Serve hits straight from the cache. + if data, ok := c.csc.Get(ctx, key); ok { + if err := ctx.Err(); err != nil { + return err + } + if err := applyCachedReply(cmd, data); isCacheableReplyResult(err) { + return err + } + c.csc.DeleteByCacheKey(key) + } + + token, shouldFetch := c.csc.Reserve(key, nsRedisKeys) + if !shouldFetch { + // Another goroutine is fetching; Get below waits until it completes. + if data, ok := c.csc.Get(ctx, key); ok { + if err := ctx.Err(); err != nil { + return err + } + if err := applyCachedReply(cmd, data); isCacheableReplyResult(err) { + return err + } + c.csc.DeleteByCacheKey(key) + } + // Original fetcher cancelled or its value was invalidated; try to take + // over so later waiters still benefit from the cache. + token, shouldFetch = c.csc.Reserve(key, nsRedisKeys) + } + + var fc cscFetchCapture + var capture *cscFetchCapture + if shouldFetch { + capture = &fc + // Release the placeholder if processWithRetry panics; Cancel on a + // stale token is a no-op. + defer func() { + if capture != nil { + c.csc.Cancel(key, token) + } + }() + } + + err := c.processWithRetry(ctx, cmd, capture, state) + + if shouldFetch { + capture = nil // disarm the deferred Cancel + if isCacheableReplyResult(err) { + c.fulfillCached(key, token, &fc) + } else { + c.csc.Cancel(key, token) + } + } + return err +} + +// fulfillCached stores a fetched value, attributing it to its serving conn when +// an evict-on-remove hook is active so EvictByConn can drop it if that conn is +// removed. It also closes the attribute-vs-coverage races: the conn is released +// before this runs, so its OnRemove eviction — or a handoff re-init's scoped +// eviction — may fire before the entry exists. Publication is serialized with +// the hook's init-generation changes, so a reply whose invalidation coverage +// was already lost never becomes visible and never wakes waiters with stale +// data. +func (c *baseClient) fulfillCached(key string, token uint64, fc *cscFetchCapture) bool { + if active := c.cscActive; active != nil && !active.Load() { + c.csc.Cancel(key, token) + return false + } + if hook := c.cscHook(); hook != nil { + if fc.connID == 0 { + // Invariant: an active hook always gets a real conn id (>=1). A zero id + // would leave the entry unattributed and un-evictable, so fail closed. + c.csc.Cancel(key, token) + return false + } + if !hook.fulfillOwnedIfCovered(key, token, fc.connID, fc.initGen, fc.raw) { + // A coverage mismatch leaves the reservation IN_PROGRESS because + // FulfillOwned was deliberately skipped. Cancel wakes its waiters + // as misses so one can safely refetch on a covered connection. + c.csc.Cancel(key, token) + return false + } + return true + } + return c.csc.FulfillOwned(key, token, 0, fc.raw) +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/csc_stats.go b/backend/vendor/github.com/redis/go-redis/v9/csc_stats.go new file mode 100644 index 00000000..d14d3fcf --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/csc_stats.go @@ -0,0 +1,36 @@ +package redis + +// CSCStats reports cumulative client-side cache activity and current +// residency. +// +// Experimental: this API may change in a minor release. +type CSCStats struct { + Hits uint64 + Misses uint64 + Entries int + MemoryUsageBytes int64 +} + +// cacheStatsReporter is an optional interface a Cache implementation may +// satisfy to expose statistics. The built-in LocalCache does; user +// implementations are not required to. +type cacheStatsReporter interface { + Stats() CSCStats +} + +// CSCStats returns statistics for this client's client-side cache, read from +// the shared cache when its implementation exposes them (the built-in +// LocalCache does). +// +// It returns a zero value when CSC is not configured or stats are unavailable. +// +// Experimental: this API may change in a minor release. +func (c *Client) CSCStats() CSCStats { + if c == nil || c.baseClient.csc == nil { + return CSCStats{} + } + if r, ok := c.baseClient.csc.(cacheStatsReporter); ok { + return r.Stats() + } + return CSCStats{} +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/docker-compose.yml b/backend/vendor/github.com/redis/go-redis/v9/docker-compose.yml index fed908be..de9683a6 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/docker-compose.yml +++ b/backend/vendor/github.com/redis/go-redis/v9/docker-compose.yml @@ -1,6 +1,6 @@ --- -x-default-image: &default-image ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.8.0} +x-default-image: &default-image ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.10.0} services: redis: diff --git a/backend/vendor/github.com/redis/go-redis/v9/error.go b/backend/vendor/github.com/redis/go-redis/v9/error.go index 06ecca74..ad3a81b7 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/error.go +++ b/backend/vendor/github.com/redis/go-redis/v9/error.go @@ -168,6 +168,9 @@ func shouldRetry(err error, retryTimeout bool) bool { return true } + // Other server errors are not retried. This includes the logical + // -SEARCH_TIMEOUT (search-on-timeout fail): retrying would just repeat the + // same expensive query. return false } diff --git a/backend/vendor/github.com/redis/go-redis/v9/generic_commands.go b/backend/vendor/github.com/redis/go-redis/v9/generic_commands.go index c7100222..6d6d8e77 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/generic_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/generic_commands.go @@ -129,6 +129,9 @@ func (c cmdable) ExpireAt(ctx context.Context, key string, tm time.Time) *BoolCm return cmd } +// ExpireTime returns the absolute expiration time of key as a Unix timestamp +// encoded in *DurationCmd (seconds since the epoch), not a remaining TTL. +// Convert with: time.Unix(int64(d/time.Second), 0). Use TTL/PTTL for remaining TTL. func (c cmdable) ExpireTime(ctx context.Context, key string) *DurationCmd { cmd := NewDurationCmd(ctx, time.Second, "expiretime", key) _ = c(ctx, cmd) @@ -209,6 +212,9 @@ func (c cmdable) PExpireAt(ctx context.Context, key string, tm time.Time) *BoolC return cmd } +// PExpireTime returns the absolute expiration time of key as a Unix timestamp +// encoded in *DurationCmd (milliseconds since the epoch), not a remaining TTL. +// Convert with: time.UnixMilli(int64(d/time.Millisecond)). Use TTL/PTTL for remaining TTL. func (c cmdable) PExpireTime(ctx context.Context, key string) *DurationCmd { cmd := NewDurationCmd(ctx, time.Millisecond, "pexpiretime", key) _ = c(ctx, cmd) diff --git a/backend/vendor/github.com/redis/go-redis/v9/hash_commands.go b/backend/vendor/github.com/redis/go-redis/v9/hash_commands.go index 256b8746..3174eab7 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/hash_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/hash_commands.go @@ -44,6 +44,11 @@ type HashCmdable interface { HPExpireTime(ctx context.Context, key string, fields ...string) *IntSliceCmd HTTL(ctx context.Context, key string, fields ...string) *IntSliceCmd HPTTL(ctx context.Context, key string, fields ...string) *IntSliceCmd + // note: the HIMPORT API is experimental and may be subject to change. + HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd + HImportSet(ctx context.Context, key, fieldsetName string, values ...interface{}) *StatusCmd + HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd + HImportDiscardAll(ctx context.Context) *IntCmd } func (c cmdable) HDel(ctx context.Context, key string, fields ...string) *IntCmd { diff --git a/backend/vendor/github.com/redis/go-redis/v9/himport.go b/backend/vendor/github.com/redis/go-redis/v9/himport.go new file mode 100644 index 00000000..a1d9125a --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/himport.go @@ -0,0 +1,473 @@ +package redis + +import ( + "context" + "strings" + "sync" + + "github.com/redis/go-redis/v9/internal" + "github.com/redis/go-redis/v9/internal/pool" + "github.com/redis/go-redis/v9/internal/proto" +) + +// himportFieldset is the client-side record of a fieldset registered with +// HImportPrepare. +type himportFieldset struct { + fields []string + version uint64 +} + +// himportRegistry remembers fieldsets registered through a client so HIMPORT +// SET can lazily prepare them on whichever pooled connection it executes. +// Versions increase monotonically and start at 1; re-registering a name under +// a new version invalidates every connection's prepared flag for it, so a +// replaced fieldset is re-prepared before its next use. +// +// Discards propagate lazily as well: a discarded name is kept as a tombstone +// and the discard-all counter as an epoch, and connections whose sessions +// still hold discarded fieldsets replay HIMPORT DISCARD/DISCARDALL before +// their next HIMPORT command (see baseClient.himportInjectedCmds). +type himportRegistry struct { + mu sync.RWMutex + nextVersion uint64 + fieldsets map[string]himportFieldset + // tombstones holds names discarded through this client whose server-side + // copies may survive on pooled connections that prepared them. An entry + // is removed when the name is registered again (the new version replaces + // the fieldset on the server, so no discard is needed) or by discardAll. + // Known limitation: a workload discarding many uniquely-named fieldsets + // grows this map for the client's lifetime and pays an O(tombstones) + // snapshot per HIMPORT round trip; HImportDiscardAll resets it. + tombstones map[string]struct{} + // discardAllEpoch increments on every successful HImportDiscardAll. + discardAllEpoch uint64 +} + +func newHImportRegistry() *himportRegistry { + return &himportRegistry{} +} + +// register stores the fieldset and returns its new version together with the +// current discard-all epoch. +func (r *himportRegistry) register(name string, fields []string) (version, epoch uint64) { + r.mu.Lock() + defer r.mu.Unlock() + if r.fieldsets == nil { + r.fieldsets = make(map[string]himportFieldset) + } + delete(r.tombstones, name) + r.nextVersion++ + r.fieldsets[name] = himportFieldset{ + fields: append([]string(nil), fields...), + version: r.nextVersion, + } + return r.nextVersion, r.discardAllEpoch +} + +func (r *himportRegistry) lookup(name string) (himportFieldset, bool) { + if r == nil { + return himportFieldset{}, false + } + r.mu.RLock() + fs, ok := r.fieldsets[name] + r.mu.RUnlock() + return fs, ok +} + +// discard removes the fieldset and leaves a tombstone so connections whose +// sessions still hold it replay the DISCARD before their next HIMPORT +// command. It reports whether the fieldset was registered. +func (r *himportRegistry) discard(name string) bool { + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.fieldsets[name]; !ok { + return false + } + delete(r.fieldsets, name) + if r.tombstones == nil { + r.tombstones = make(map[string]struct{}) + } + r.tombstones[name] = struct{}{} + return true +} + +// discardAll drops every fieldset and tombstone and moves to a new epoch; +// connections prepared under an older epoch replay HIMPORT DISCARDALL before +// their next HIMPORT command. It returns the new epoch and the number of +// fieldsets that were registered. +func (r *himportRegistry) discardAll() (epoch uint64, removed int) { + r.mu.Lock() + removed = len(r.fieldsets) + r.fieldsets = nil + r.tombstones = nil + r.discardAllEpoch++ + epoch = r.discardAllEpoch + r.mu.Unlock() + return epoch, removed +} + +// discardVersion withdraws a registration whose fan-out PREPARE was rejected +// by a server — but only while the entry is still at that version, so a +// concurrent re-registration is not clobbered. A tombstone is left: the +// fan-out may have succeeded on some masters before another rejected it +// (per-node ACLs, rolling upgrades), and those sessions hold the withdrawn +// fieldset; the tombstone makes their next HIMPORT command discard it +// instead of leaving a fieldset the client can no longer address. +func (r *himportRegistry) discardVersion(name string, version uint64) { + r.mu.Lock() + if fs, ok := r.fieldsets[name]; ok && fs.version == version { + delete(r.fieldsets, name) + if r.tombstones == nil { + r.tombstones = make(map[string]struct{}) + } + r.tombstones[name] = struct{}{} + } + r.mu.Unlock() +} + +// refreshVersion bumps a registered fieldset to a new version, keeping its +// fields — but only while the entry is still at the given version, so a +// concurrent re-registration is not disturbed. Every connection's prepared +// flag becomes stale, forcing a re-prepare before the fieldset's next use on +// each of them. Used when a "no such fieldset" reply signals session loss +// that may have hit more connections than the one that reported it (failover, +// cross-region switch, reset storms). +func (r *himportRegistry) refreshVersion(name string, version uint64) { + r.mu.Lock() + if fs, ok := r.fieldsets[name]; ok && fs.version == version { + r.nextVersion++ + fs.version = r.nextVersion + r.fieldsets[name] = fs + } + r.mu.Unlock() +} + +// idle reports whether the registry implies no injection work at all: no +// fieldsets to replay, no tombstones to discard, and no discard-all epoch a +// session could be behind. +func (r *himportRegistry) idle() bool { + if r == nil { + return true + } + r.mu.RLock() + idle := len(r.fieldsets) == 0 && len(r.tombstones) == 0 && r.discardAllEpoch == 0 + r.mu.RUnlock() + return idle +} + +// cleanupSnapshot returns the current epoch and the tombstoned names. +func (r *himportRegistry) cleanupSnapshot() (epoch uint64, tombstones []string) { + r.mu.RLock() + epoch = r.discardAllEpoch + if len(r.tombstones) > 0 { + tombstones = make([]string, 0, len(r.tombstones)) + for name := range r.tombstones { + tombstones = append(tombstones, name) + } + } + r.mu.RUnlock() + return epoch, tombstones +} + +// himportNoSuchFieldset reports whether err is the server's "no such +// fieldset" reply, i.e. an HIMPORT SET executed on a connection whose session +// does not hold the referenced fieldset. +func himportNoSuchFieldset(err error) bool { + return isRedisError(err) && strings.Contains(err.Error(), "no such fieldset") +} + +// himportInjectedCmds returns the HIMPORT commands to write to cn ahead of a +// batch, in order: +// +// 1. HIMPORT DISCARDALL when cn's session was prepared under an older +// discard-all epoch; +// 2. HIMPORT DISCARD for each discarded fieldset the session still holds; +// 3. HIMPORT PREPARE for each registered fieldset referenced by an HIMPORT +// SET in the batch that the session lacks at the current version. +// +// A fieldset covered by a user-issued PREPARE earlier in the batch needs no +// injection — the server session holds it by the time the SET runs. Returns +// nil when the batch contains no HIMPORT commands: sessions holding only +// discarded fieldsets are cleaned up on their next HIMPORT use, not on +// unrelated traffic. +func (c *baseClient) himportInjectedCmds(ctx context.Context, cn *pool.Conn, cmds []Cmder) []Cmder { + if c.himport.idle() { + return nil + } + hasHImport := false + for _, cmd := range cmds { + if _, ok := cmd.(himportCmder); ok { + hasHImport = true + break + } + } + if !hasHImport { + return nil + } + + var injected []Cmder + + // Discards first: a session behind the discard-all epoch is wiped + // entirely; otherwise individual tombstoned fieldsets it still holds are + // discarded. + epoch, tombstones := c.himport.cleanupSnapshot() + sessionWiped := false + if cn.HasPreparedFieldsets() && cn.FieldsetEpoch() != epoch { + da := NewHImportDiscardAllCmd(ctx) + da.registryEpoch = epoch + injected = append(injected, da) + sessionWiped = true + } else { + for _, name := range tombstones { + if cn.FieldsetPreparedVersion(name) != 0 { + injected = append(injected, NewHImportDiscardCmd(ctx, name)) + } + } + } + + // Prepares for registered fieldsets the batch's SETs reference. + var covered map[string]struct{} + cover := func(name string) { + if covered == nil { + covered = make(map[string]struct{}) + } + covered[name] = struct{}{} + } + for _, cmd := range cmds { + switch hc := cmd.(type) { + case *HImportPrepareCmd: + cover(hc.fieldsetName) + case *HImportSetCmd: + if _, ok := covered[hc.fieldsetName]; ok { + continue + } + fs, ok := c.himport.lookup(hc.fieldsetName) + if !ok { + continue + } + if !sessionWiped && cn.FieldsetPreparedVersion(hc.fieldsetName) == fs.version { + continue + } + // The session holds an older version. Discard it before the + // re-prepare: the SET behind it is already on the wire, and if + // the re-prepare fails the SET must answer "no such fieldset" + // rather than silently writing the old version's field names. + if !sessionWiped && cn.FieldsetPreparedVersion(hc.fieldsetName) != 0 { + injected = append(injected, NewHImportDiscardCmd(ctx, hc.fieldsetName)) + } + prep := NewHImportPrepareCmd(ctx, hc.fieldsetName, fs.fields...) + prep.registryVersion = fs.version + prep.registryEpoch = epoch + injected = append(injected, prep) + cover(hc.fieldsetName) + } + } + return injected +} + +// himportReadInjectedReplies consumes the replies of injected HIMPORT +// commands. Server errors are recorded on the command and the connection is +// left readable; transport errors are returned. Successful commands apply +// their prepared-flag bookkeeping on cn. +func (c *baseClient) himportReadInjectedReplies(ctx context.Context, cn *pool.Conn, rd *proto.Reader, injected []Cmder) error { + for _, cmd := range injected { + if err := c.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) + } + err := cmd.readReply(rd) + cmd.SetErr(err) + if err != nil { + if !isRedisError(err) { + return err + } + // A failed injected PREPARE becomes the root cause of the + // dependent SETs' errors downstream; a failed injected discard + // only delays cleanup until the next HIMPORT command. + internal.Logger.Printf(ctx, "himport: injected %s failed: %v", cmd.Name(), err) + continue + } + switch hc := cmd.(type) { + case *HImportPrepareCmd: + cn.MarkFieldsetPrepared(hc.fieldsetName, hc.registryVersion, hc.registryEpoch) + case *HImportDiscardCmd: + cn.UnmarkFieldsetPrepared(hc.fieldsetName) + case *HImportDiscardAllCmd: + cn.ClearPreparedFieldsets(hc.registryEpoch) + } + } + return nil +} + +// himportAfterCmd applies registry and prepared-flag updates after a +// user-issued HIMPORT command completed successfully on cn. +func (c *baseClient) himportAfterCmd(cn *pool.Conn, hc himportCmder) { + if c.himport == nil { + return + } + switch cmd := hc.(type) { + case *HImportPrepareCmd: + version, epoch := cmd.registryVersion, cmd.registryEpoch + if version == 0 { + version, epoch = c.himport.register(cmd.fieldsetName, cmd.fields) + } + // A pre-assigned version marks a fan-out copy: the fieldset was + // registered once at the cluster/ring level; only mark the + // executing connection. + cn.MarkFieldsetPrepared(cmd.fieldsetName, version, epoch) + case *HImportDiscardCmd: + registered := c.himport.discard(cmd.fieldsetName) + cn.UnmarkFieldsetPrepared(cmd.fieldsetName) + // The managed API reports the registry lifecycle: 1 when the + // fieldset was registered on this client and is now removed. The + // executing connection's session count stands only for fieldsets + // the registry never knew (raw usage). + if registered { + cmd.SetVal(1) + } + case *HImportDiscardAllCmd: + // A pre-assigned epoch marks a fan-out copy: the registry was + // already wiped at the cluster/ring level; only move the executing + // connection to that epoch. + if cmd.registryEpoch != 0 { + cn.ClearPreparedFieldsets(cmd.registryEpoch) + return + } + epoch, removed := c.himport.discardAll() + cn.ClearPreparedFieldsets(epoch) + // Same registry semantics: report how many registered fieldsets + // were removed, not how many the executing session happened to + // hold. + if removed > 0 { + cmd.SetVal(int64(removed)) + } + } +} + +// himportAfterBatch runs after all replies of a batch were read: it surfaces +// an injected PREPARE failure as the root cause on the HIMPORT SET commands +// that depended on it (their own reply is the secondary "no such fieldset" +// error), invalidates stale prepared flags for SETs that found their +// registered fieldset missing server-side, and applies registry updates for +// user-issued HIMPORT commands that succeeded in the batch. +// rawErr throughout: this runs on the execution path, before an async +// autopipeline batch completes (its ready channel closes only after the +// pipeline hook chain returns) — Err() on a user command would await and +// self-deadlock the dispatcher. +func (c *baseClient) himportAfterBatch(cn *pool.Conn, injected []Cmder, cmds []Cmder) { + var failed map[string]error + var refreshed map[string]struct{} + for _, cmd := range injected { + if prep, ok := cmd.(*HImportPrepareCmd); ok { + if err := prep.Err(); err != nil { + if failed == nil { + failed = make(map[string]error) + } + failed[prep.fieldsetName] = err + } + } + } + for _, cmd := range cmds { + hc, ok := cmd.(himportCmder) + if !ok { + continue + } + if set, ok := hc.(*HImportSetCmd); ok { + if rootCause, ok := failed[set.fieldsetName]; ok && himportNoSuchFieldset(set.rawErr()) { + set.SetErr(rootCause) + continue + } + // The session lost a fieldset the flags claim is prepared (e.g. + // RESET) — and the same event may have wiped other sessions + // whose flags also still look current. Bump the fieldset + // version once so the SET's re-issue, the cluster re-queue on + // whichever connection it lands, or the caller's transaction + // retry replays the PREPARE. + if himportNoSuchFieldset(set.rawErr()) { + if _, done := refreshed[set.fieldsetName]; !done { + if refreshed == nil { + refreshed = make(map[string]struct{}) + } + refreshed[set.fieldsetName] = struct{}{} + if fs, registered := c.himport.lookup(set.fieldsetName); registered { + c.himport.refreshVersion(set.fieldsetName, fs.version) + } + } + } + continue + } + if hc.rawErr() == nil { + c.himportAfterCmd(cn, hc) + } + } +} + +// himportRetryFailedSets re-issues, once, the HIMPORT SET commands of a +// pipeline batch that failed with "no such fieldset" while their fieldset is +// registered — the error must not surface for managed fieldsets (NF.4). Only +// the SETs are re-sent: HIMPORT SET is a full replace, so re-execution is +// idempotent, and no other command of the batch runs again. Their prepared +// flags were invalidated by himportAfterBatch, so himportInjectedCmds +// regenerates the PREPAREs for this connection. Transport errors are +// returned; server errors stay recorded on the commands. +// (The retry does not carry an ASKING prefix. A redirected [ASKING, SET] +// pair whose injected PREPARE failed is excluded by the root-cause swap in +// himportAfterBatch; one that lost its session without an injection can be +// re-issued here, and the bare SET then draws a fresh MOVED/ASK that the +// outer cluster redirect handling resolves.) +func (c *baseClient) himportRetryFailedSets(ctx context.Context, cn *pool.Conn, cmds []Cmder) error { + if c.himport.idle() { + return nil + } + var retry []Cmder + for _, cmd := range cmds { + // rawErr: execution path, same self-deadlock rule as himportAfterBatch. + if set, ok := cmd.(*HImportSetCmd); ok && himportNoSuchFieldset(set.rawErr()) { + if _, registered := c.himport.lookup(set.fieldsetName); registered { + retry = append(retry, set) + } + } + } + if len(retry) == 0 { + return nil + } + + injected := c.himportInjectedCmds(ctx, cn, retry) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } + return writeCmds(wr, retry) + }); err != nil { + return err + } + return cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { + if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + return err + } + err := c.pipelineReadCmds(ctx, cn, rd, retry) + if err != nil && !isRedisError(err) { + return err + } + // Server errors (including a repeated failure) stay on the + // individual commands; the batch as a whole is done. + c.himportAfterBatch(cn, injected, retry) + return nil + }) +} + +// himportShouldRetrySet reports whether a retry of cmd may succeed after it +// failed with "no such fieldset": true when the fieldset is registered +// client-side — the executing connection lost its server session state (for +// example a RESET, or a concurrent discard). The connection's prepared flag +// was already invalidated inside _process, while that goroutine still owned +// the connection, so the retry re-prepares lazily wherever it lands. +func (c *baseClient) himportShouldRetrySet(cmd Cmder, err error) bool { + set, ok := cmd.(*HImportSetCmd) + if !ok || !himportNoSuchFieldset(err) { + return false + } + _, registered := c.himport.lookup(set.fieldsetName) + return registered +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/himport_cluster.go b/backend/vendor/github.com/redis/go-redis/v9/himport_cluster.go new file mode 100644 index 00000000..57cddb0d --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/himport_cluster.go @@ -0,0 +1,206 @@ +package redis + +import "context" + +// Cluster and ring support for the HIMPORT command family. +// +// Correctness comes from the shared registry: every node/shard client holds +// the same himportRegistry (wired at client construction), so any connection +// executing an HIMPORT SET lazily replays the PREPARE, MOVED/ASK redirects +// re-prepare on the target node, and discards propagate through tombstones +// and the discard-all epoch. Replicas share the registry too — roles change +// with the topology, and a promoted replica's connections simply carry no +// prepared flags, so their first SET self-prepares. +// +// On top of that, user-issued PREPARE/DISCARD/DISCARDALL fan out eagerly to +// all masters (R.4): one connection per master is prepared/cleaned up front, +// server-side validation surfaces immediately, and leftover session state is +// bounded. The fan-out is best-effort — any connection it does not reach is +// covered by the lazy replay. + +// The fan-out helpers execute the per-node copies through each node client's +// Process, so node-level hooks observe them; the cluster/ring-level +// ProcessHook chain sees only the user's command object, not the fan-out. +// +// Known limitation: an HImportPrepare pipelined together with HImportSets of +// the same new fieldset in one ClusterClient Exec is not ordered across +// nodes — per-node sub-batches run concurrently, and the registration +// happens when the PREPARE's node completes, so SETs routed to other nodes +// can race it and fail with "no such fieldset". Register the fieldset with +// the client-level HImportPrepare before pipelining (the HLD's back-to-back +// PREPARE+SET pattern is a single-connection guarantee). + +// himportForEach runs fn on a set of clients (all cluster masters, or all +// ring shards). +type himportForEach func(ctx context.Context, fn func(ctx context.Context, client *Client) error) error + +// himportRequeueFailedSets re-queues HIMPORT SETs of registered fieldsets +// that failed with "no such fieldset" — their stale prepared flags were just +// invalidated by himportAfterBatch, so the next pipeline attempt re-prepares +// lazily and re-executes only those SETs (a full replace, so idempotent). +// Bounded by the cluster pipeline's attempt budget. +func (c *ClusterClient) himportRequeueFailedSets(ctx context.Context, cmds []Cmder, failedCmds *cmdsMap) { + for _, cmd := range cmds { + // rawErr: runs on the per-node execution goroutine, same + // self-deadlock rule as himportAfterBatch. + if set, ok := cmd.(*HImportSetCmd); ok && himportNoSuchFieldset(set.rawErr()) { + if _, registered := c.himport.lookup(set.fieldsetName); registered { + _ = c.mapCmdsByNode(ctx, failedCmds, []Cmder{set}) + } + } + } +} + +// himportFanOutPrepare registers the fieldset once in the shared registry +// and executes a pre-versioned PREPARE copy on every client; each copy marks +// its executing connection without registering again. A deterministic server +// rejection (e.g. duplicate field name) withdraws the registration; a +// transport failure keeps it, and lazy replay covers the connections the +// fan-out missed (all-succeeded semantics: the first error is reported). +func himportFanOutPrepare(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportPrepareCmd) { + version, epoch := registry.register(cmd.fieldsetName, cmd.fields) + err := forEach(ctx, func(ctx context.Context, client *Client) error { + fanCmd := NewHImportPrepareCmd(ctx, cmd.fieldsetName, cmd.fields...) + fanCmd.registryVersion = version + fanCmd.registryEpoch = epoch + return client.Process(ctx, fanCmd) + }) + if err != nil { + if isRedisError(err) { + // Withdraw the registration; the tombstone cleans the sessions + // on which the fan-out succeeded before the rejection. + registry.discardVersion(cmd.fieldsetName, version) + } + cmd.SetErr(err) + return + } + cmd.SetVal("OK") +} + +// himportFanOutDiscard removes the fieldset from the shared registry +// (leaving the tombstone that lazily cleans the connections the fan-out does +// not reach) and discards it on one connection of every client. +func himportFanOutDiscard(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportDiscardCmd) { + registered := registry.discard(cmd.fieldsetName) + err := forEach(ctx, func(ctx context.Context, client *Client) error { + return client.Process(ctx, NewHImportDiscardCmd(ctx, cmd.fieldsetName)) + }) + if err != nil { + cmd.SetErr(err) + return + } + if registered { + cmd.SetVal(1) + } else { + cmd.SetVal(0) + } +} + +// himportFanOutDiscardAll wipes the shared registry once and executes a +// pre-epoch DISCARDALL copy on every client; each copy moves its executing +// connection to the new epoch without bumping the registry again. +func himportFanOutDiscardAll(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportDiscardAllCmd) { + epoch, removed := registry.discardAll() + err := forEach(ctx, func(ctx context.Context, client *Client) error { + fanCmd := NewHImportDiscardAllCmd(ctx) + fanCmd.registryEpoch = epoch + return client.Process(ctx, fanCmd) + }) + if err != nil { + cmd.SetErr(err) + return + } + cmd.SetVal(int64(removed)) +} + +// HImportPrepare registers the fieldset in the cluster-wide registry and +// eagerly prepares one connection on every master; all other connections — +// including those of replicas promoted later and masters added by +// resharding — are prepared lazily before their first HImportSet. See +// HashCmdable.HImportPrepare (cmdable) for the fieldset semantics. +// +// The fan-out is best-effort and reports the first error: on a server +// rejection (e.g. duplicate field name) the registration is withdrawn and +// any sessions the fan-out already prepared are cleaned lazily; on a +// transport failure the registration is kept and lazy replay covers the +// connections the fan-out missed. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *ClusterClient) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd { + cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...) + himportFanOutPrepare(ctx, c.himport, c.ForEachMaster, cmd) + return &cmd.StatusCmd +} + +// HImportDiscard removes the fieldset from the cluster-wide registry and +// discards it on every master; connections the fan-out does not reach +// replay the discard before their next HIMPORT command. It returns 1 if the +// fieldset was registered on this client and is now removed. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *ClusterClient) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd { + cmd := NewHImportDiscardCmd(ctx, fieldsetName) + himportFanOutDiscard(ctx, c.himport, c.ForEachMaster, cmd) + return &cmd.IntCmd +} + +// HImportDiscardAll removes all fieldsets from the cluster-wide registry and +// wipes them on every master; connections the fan-out does not reach replay +// the wipe before their next HIMPORT command. It returns the number of +// fieldsets removed from the registry. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *ClusterClient) HImportDiscardAll(ctx context.Context) *IntCmd { + cmd := NewHImportDiscardAllCmd(ctx) + himportFanOutDiscardAll(ctx, c.himport, c.ForEachMaster, cmd) + return &cmd.IntCmd +} + +// HImportPrepare registers the fieldset in the ring-wide registry and +// eagerly prepares one connection on every shard; all other connections are +// prepared lazily before their first HImportSet. The fan-out is best-effort +// with the same failure semantics as ClusterClient.HImportPrepare. See +// HashCmdable.HImportPrepare (cmdable) for the fieldset semantics. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *Ring) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd { + cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...) + himportFanOutPrepare(ctx, c.opt.himport, c.ForEachShard, cmd) + return &cmd.StatusCmd +} + +// HImportDiscard removes the fieldset from the ring-wide registry and +// discards it on every shard; connections the fan-out does not reach replay +// the discard before their next HIMPORT command. It returns 1 if the +// fieldset was registered on this client and is now removed. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *Ring) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd { + cmd := NewHImportDiscardCmd(ctx, fieldsetName) + himportFanOutDiscard(ctx, c.opt.himport, c.ForEachShard, cmd) + return &cmd.IntCmd +} + +// HImportDiscardAll removes all fieldsets from the ring-wide registry and +// wipes them on every shard; connections the fan-out does not reach replay +// the wipe before their next HIMPORT command. It returns the number of +// fieldsets removed from the registry. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c *Ring) HImportDiscardAll(ctx context.Context) *IntCmd { + cmd := NewHImportDiscardAllCmd(ctx) + himportFanOutDiscardAll(ctx, c.opt.himport, c.ForEachShard, cmd) + return &cmd.IntCmd +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/himport_commands.go b/backend/vendor/github.com/redis/go-redis/v9/himport_commands.go new file mode 100644 index 00000000..77e1a98c --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/himport_commands.go @@ -0,0 +1,253 @@ +package redis + +import "context" + +// The HIMPORT command family (Redis 8.10+, "hinted hash templates") provides +// fast ingestion of many hashes sharing the same field names. HIMPORT PREPARE +// registers the field names once under a fieldset name, then HIMPORT SET +// creates hashes by sending only the values. +// +// The server scopes a fieldset to the physical connection that prepared it. +// Because go-redis pools connections, the client additionally keeps a +// client-side registry of fieldsets registered through HImportPrepare and +// lazily replays the PREPARE (at most once per connection session) on any +// pooled connection about to execute an HImportSet that references it. See +// himport.go. +// +// The whole HIMPORT surface — the typed methods, the HImport*Cmd types and +// their constructors — is experimental and may be subject to change. + +// himportCmder marks HIMPORT commands that participate in client-side +// fieldset tracking. Process paths do a single interface assertion on the +// hot path and inspect the concrete type only for HIMPORT commands. +type himportCmder interface { + Cmder + himportCmd() +} + +var ( + _ himportCmder = (*HImportPrepareCmd)(nil) + _ himportCmder = (*HImportSetCmd)(nil) + _ himportCmder = (*HImportDiscardCmd)(nil) + _ himportCmder = (*HImportDiscardAllCmd)(nil) +) + +// HImportPrepareCmd represents an HIMPORT PREPARE command. +type HImportPrepareCmd struct { + StatusCmd + + fieldsetName string + fields []string + + // registryVersion and registryEpoch are set only on commands injected by + // the client to replay a registered fieldset onto a connection; on + // success the connection is marked as prepared at this version under + // this discard-all epoch. + registryVersion uint64 + registryEpoch uint64 +} + +func (cmd *HImportPrepareCmd) himportCmd() {} + +// NewHImportPrepareCmd returns an HIMPORT PREPARE command. +func NewHImportPrepareCmd(ctx context.Context, fieldsetName string, fields ...string) *HImportPrepareCmd { + args := make([]interface{}, 3+len(fields)) + args[0] = "himport" + args[1] = "prepare" + args[2] = fieldsetName + for i, field := range fields { + args[3+i] = field + } + return &HImportPrepareCmd{ + StatusCmd: StatusCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: args, + cmdType: CmdTypeStatus, + }, + }, + fieldsetName: fieldsetName, + fields: append([]string(nil), fields...), + } +} + +// HImportSetCmd represents an HIMPORT SET command. +type HImportSetCmd struct { + StatusCmd + + fieldsetName string +} + +func (cmd *HImportSetCmd) himportCmd() {} + +// NewHImportSetCmd returns an HIMPORT SET command. +func NewHImportSetCmd(ctx context.Context, key, fieldsetName string, values ...interface{}) *HImportSetCmd { + args := make([]interface{}, 4+len(values)) + args[0] = "himport" + args[1] = "set" + args[2] = key + args[3] = fieldsetName + copy(args[4:], values) + cmd := &HImportSetCmd{ + StatusCmd: StatusCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: args, + cmdType: CmdTypeStatus, + }, + }, + fieldsetName: fieldsetName, + } + cmd.SetFirstKeyPos(2) + return cmd +} + +// HImportDiscardCmd represents an HIMPORT DISCARD command. +type HImportDiscardCmd struct { + IntCmd + + fieldsetName string +} + +func (cmd *HImportDiscardCmd) himportCmd() {} + +// NewHImportDiscardCmd returns an HIMPORT DISCARD command. +func NewHImportDiscardCmd(ctx context.Context, fieldsetName string) *HImportDiscardCmd { + return &HImportDiscardCmd{ + IntCmd: IntCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: []interface{}{"himport", "discard", fieldsetName}, + cmdType: CmdTypeInt, + }, + }, + fieldsetName: fieldsetName, + } +} + +// HImportDiscardAllCmd represents an HIMPORT DISCARDALL command. +type HImportDiscardAllCmd struct { + IntCmd + + // registryEpoch is set only on commands injected by the client to wipe a + // session that predates the registry's discard-all epoch; on success the + // connection adopts this epoch. + registryEpoch uint64 +} + +func (cmd *HImportDiscardAllCmd) himportCmd() {} + +// NewHImportDiscardAllCmd returns an HIMPORT DISCARDALL command. +func NewHImportDiscardAllCmd(ctx context.Context) *HImportDiscardAllCmd { + return &HImportDiscardAllCmd{ + IntCmd: IntCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: []interface{}{"himport", "discardall"}, + cmdType: CmdTypeInt, + }, + }, + } +} + +// HImportPrepare registers an ordered list of hash field names under +// fieldsetName for use by subsequent HImportSet calls: +// +// HIMPORT PREPARE fieldset_name field [field ...] +// +// The server keeps the fieldset in the session of the connection that +// executed the command. On pooled clients (Client, Conn, Pipeline, Tx) the +// fieldset is also remembered client-side and the PREPARE is replayed +// lazily — at most once per connection session — on any pooled connection +// about to execute an HImportSet referencing it, so HImportSet works +// transparently across the pool. Preparing an existing fieldset name again +// silently replaces it. +// +// ClusterClient and Ring override this method (see himport_cluster.go): the +// fieldset registers in a registry shared by every node/shard client and the +// PREPARE additionally fans out eagerly to all masters/shards. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c cmdable) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd { + cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...) + _ = c(ctx, cmd) + return &cmd.StatusCmd +} + +// HImportSet creates or fully replaces the hash at key using the field list +// registered under fieldsetName, pairing values positionally with the +// prepared fields: +// +// HIMPORT SET key fieldset_name value [value ...] +// +// The number of values must equal the fieldset's field count. The resulting +// key is a regular hash readable and writable by all hash commands. If the +// fieldset was registered through HImportPrepare on this client, it is +// prepared automatically on whichever pooled connection executes the command; +// otherwise the fieldset must have been prepared on the executing connection +// or the server replies "ERR no such fieldset". +// +// "no such fieldset" never surfaces for a registered fieldset: a +// single-command HImportSet whose connection lost its session state (e.g. +// RESET) is transparently re-prepared and retried once — the failure also +// stales every other connection's prepared flag, so the retry re-prepares +// wherever it lands, and this recovery attempt is granted even when retries +// are disabled (MaxRetries -1). In pipelines the failed HImportSets — and +// only those — are re-prepared and re-issued once on the same connection +// (HIMPORT SET is a full replace, so the re-execution is idempotent and no +// other command of the batch runs again). Inside transactions the error does +// surface after EXEC — an executed transaction cannot be partially re-run — +// but the prepared flags are invalidated, so retrying the transaction +// succeeds. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c cmdable) HImportSet(ctx context.Context, key, fieldsetName string, values ...interface{}) *StatusCmd { + cmd := NewHImportSetCmd(ctx, key, fieldsetName, values...) + _ = c(ctx, cmd) + return &cmd.StatusCmd +} + +// HImportDiscard removes fieldsetName from the executing connection's session +// and from the client-side registry, stopping further automatic replay: +// +// HIMPORT DISCARD fieldset_name +// +// It returns 1 if the fieldset was registered on this client and is now +// removed, 0 otherwise (for names never registered through the managed API, +// the executing connection's session reply passes through unchanged). Pooled +// connections whose sessions still hold the fieldset replay the DISCARD +// before their next HIMPORT command, so a subsequent HImportSet fails with +// "no such fieldset" on every connection, exactly as on a single connection. +// Hashes already created through the fieldset are not affected. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c cmdable) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd { + cmd := NewHImportDiscardCmd(ctx, fieldsetName) + _ = c(ctx, cmd) + return &cmd.IntCmd +} + +// HImportDiscardAll removes all fieldsets from the executing connection's +// session and clears the client-side registry: +// +// HIMPORT DISCARDALL +// +// It returns the number of fieldsets removed from the client-side registry +// (when none were registered, the executing connection's session count +// passes through). Other pooled connections whose sessions were prepared +// earlier replay HIMPORT DISCARDALL before their next HIMPORT command. +// +// Requires Redis 8.10 or newer. +// +// note: the API is experimental and may be subject to change. +func (c cmdable) HImportDiscardAll(ctx context.Context) *IntCmd { + cmd := NewHImportDiscardAllCmd(ctx) + _ = c(ctx, cmd) + return &cmd.IntCmd +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/once.go b/backend/vendor/github.com/redis/go-redis/v9/internal/once.go index b81244fd..2d3a8cdc 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/once.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/once.go @@ -27,7 +27,7 @@ import ( // and is re-armed on failure. type Once struct { m sync.Mutex - done uint32 + done atomic.Uint32 } // Do calls the function f if and only if Do has not been invoked @@ -46,17 +46,17 @@ type Once struct { // // err := config.once.Do(func() error { return config.init(filename) }) func (o *Once) Do(f func() error) error { - if atomic.LoadUint32(&o.done) == 1 { + if o.done.Load() == 1 { return nil } // Slow-path. o.m.Lock() defer o.m.Unlock() var err error - if o.done == 0 { + if o.done.Load() == 0 { err = f() if err == nil { - atomic.StoreUint32(&o.done, 1) + o.done.Store(1) } } return err diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/otel/metrics.go b/backend/vendor/github.com/redis/go-redis/v9/internal/otel/metrics.go index a3f23fff..2e234ba8 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/otel/metrics.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/otel/metrics.go @@ -155,6 +155,15 @@ func getRecorder() Recorder { return r } +// Enabled reports whether a real recorder is installed. Callers use it to +// skip metric work whose INPUTS are expensive to obtain — e.g. reading a +// command's result, which on the async autopipeline face blocks until the +// command executes. +func Enabled() bool { + _, noop := getRecorder().(noopRecorder) + return !noop +} + // SetGlobalRecorder sets the global recorder (called by Init() in extra/redisotel-native) func SetGlobalRecorder(r Recorder) { recorderMu.Lock() @@ -266,10 +275,15 @@ func (noopRecorder) RecordStreamLag(context.Context, time.Duration, *pool.Conn, func (noopRecorder) RecordConnectionCount(context.Context, int, *pool.Conn, string, bool) {} func (noopRecorder) RecordPendingRequests(context.Context, int, *pool.Conn, string) {} -// RegisterPools registers connection pools with the global recorder. -func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, addr string) { - // Check if the global recorder implements PoolRegistrar - if registrar, ok := globalRecorder.(PoolRegistrar); ok { +// RegisterPools registers connection pools with the global recorder. pipelinePool +// is the optional dedicated pipeline connection pool (nil when not configured); +// it is registered as a regular pool under a "_pipeline" name suffix. +func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, pipelinePool pool.Pooler, addr string) { + // Check if the global recorder implements PoolRegistrar. Read it through + // getRecorder: SetGlobalRecorder writes globalRecorder under recorderMu, and + // clients are created (and closed) concurrently with telemetry being + // installed, so an unlocked read here is a data race -race reports. + if registrar, ok := getRecorder().(PoolRegistrar); ok { // Generate a unique ID for this client's pools uniqueID := generateUniqueID() @@ -281,18 +295,27 @@ func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, addr string) { poolName := addr + "_" + uniqueID + "_pubsub" registrar.RegisterPubSubPool(poolName, pubSubPool) } + if pipelinePool != nil { + poolName := addr + "_" + uniqueID + "_pipeline" + registrar.RegisterPool(poolName, pipelinePool) + } } } -// UnregisterPools removes connection pools from the global recorder -func UnregisterPools(connPool pool.Pooler, pubSubPool PubSubPooler) { - // Check if the global recorder implements PoolRegistrar - if registrar, ok := globalRecorder.(PoolRegistrar); ok { +// UnregisterPools removes connection pools from the global recorder. pipelinePool +// is the optional dedicated pipeline connection pool (nil when not configured). +func UnregisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, pipelinePool pool.Pooler) { + // Check if the global recorder implements PoolRegistrar (see RegisterPools + // for why this goes through getRecorder rather than reading directly). + if registrar, ok := getRecorder().(PoolRegistrar); ok { if connPool != nil { registrar.UnregisterPool(connPool) } if pubSubPool != nil { registrar.UnregisterPubSubPool(pubSubPool) } + if pipelinePool != nil { + registrar.UnregisterPool(pipelinePool) + } } } diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn.go b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn.go index 1e836ad4..80f3b77c 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn.go @@ -45,7 +45,7 @@ func GetCachedTimeNs() int64 { } // Global atomic counter for connection IDs -var connIDCounter uint64 +var connIDCounter atomic.Uint64 // HandoffState represents the atomic state for connection handoffs // This struct is stored atomically to prevent race conditions between @@ -63,7 +63,7 @@ type atomicNetConn struct { // generateConnID generates a fast unique identifier for a connection with zero allocations func generateConnID() uint64 { - return atomic.AddUint64(&connIDCounter, 1) + return connIDCounter.Add(1) } type Conn struct { @@ -109,6 +109,18 @@ type Conn struct { expiresAt time.Time poolName string // Name of the pool this connection belongs to (for metrics) + // preparedFieldsets tracks HIMPORT fieldsets prepared on this + // connection's current server session: fieldset name -> client-side + // registry version. The server drops fieldsets when the session ends, + // so the map is cleared whenever the underlying network connection is + // replaced. preparedFieldsetsEpoch records the registry's discard-all + // epoch the session was prepared under; a session behind the current + // epoch replays HIMPORT DISCARDALL before its next HIMPORT command. + // Guarded by preparedFieldsetsMu; the map is nil until first use. + preparedFieldsetsMu sync.Mutex + preparedFieldsets map[string]uint64 + preparedFieldsetsEpoch uint64 + // When a goroutine closes a connection, it usually knows the reason, so closeReason is not needed. // closeReason is only used when an in-use connection is closed by another goroutine, // to inform the goroutine using the connection why the connection was closed. @@ -135,6 +147,24 @@ type Conn struct { initConnFunc func(context.Context, *Conn) error onClose func() error + + // onCscClose is the client-side-caching close hook, kept separate from + // onClose (streaming-credentials cleanup) so neither clobbers the other. + // Both keep overwrite semantics, so re-running initConn can't accumulate them. + onCscClose func() error + + // onCscReinit runs after the connection is claimed for reinitialization but + // before its socket is replaced. CSC uses it to invalidate entries whose + // server-side tracking coverage belongs to the old socket. + onCscReinit func() + + // cscReadPending requests one conservative drain after a command read through + // a transport whose buffered state cannot be fully observed by MaybeHasData. + cscReadPending atomic.Bool + + // lastCscPeriodicProbeNs throttles bounded fallback reads on platforms and + // opaque transports without a non-consuming readiness mechanism. + lastCscPeriodicProbeNs atomic.Int64 } func NewConn(netConn net.Conn) *Conn { @@ -638,6 +668,18 @@ func (cn *Conn) SetOnClose(fn func() error) { cn.onClose = fn } +// SetOnCscClose sets the client-side-caching close hook, overwriting any +// previous one. It runs on Close in addition to the SetOnClose callback. +func (cn *Conn) SetOnCscClose(fn func() error) { + cn.onCscClose = fn +} + +// SetOnCscReinit sets the client-side-caching pre-reinitialization hook, +// overwriting any previous one. +func (cn *Conn) SetOnCscReinit(fn func()) { + cn.onCscReinit = fn +} + // SetInitConnFunc sets the connection initialization function to be called on reconnections. func (cn *Conn) SetInitConnFunc(fn func(context.Context, *Conn) error) { cn.initConnFunc = fn @@ -661,6 +703,85 @@ func (cn *Conn) SetNetConn(netConn net.Conn) { cn.readerMu.Unlock() cn.bw.Reset(netConn) + + // A new socket is a new server session with no HIMPORT fieldsets and + // nothing left to discard. + cn.ClearPreparedFieldsets(0) +} + +// FieldsetPreparedVersion returns the client-side registry version at which +// the named HIMPORT fieldset was prepared on this connection's current server +// session, or 0 if it was not prepared on it (registry versions start at 1). +func (cn *Conn) FieldsetPreparedVersion(name string) uint64 { + cn.preparedFieldsetsMu.Lock() + version := cn.preparedFieldsets[name] + cn.preparedFieldsetsMu.Unlock() + return version +} + +// MarkFieldsetPrepared records that the named HIMPORT fieldset was prepared +// on this connection's current server session at the given registry version. +// A session acquiring its first fieldset adopts the given discard-all epoch +// (fieldsets prepared after an HIMPORT DISCARDALL are not subject to it); +// the epoch never moves backwards, so a mark carrying an older snapshot +// cannot regress a session already wiped at a newer epoch. +func (cn *Conn) MarkFieldsetPrepared(name string, version, epoch uint64) { + cn.preparedFieldsetsMu.Lock() + if len(cn.preparedFieldsets) == 0 { + cn.preparedFieldsets = make(map[string]uint64) + if epoch > cn.preparedFieldsetsEpoch { + cn.preparedFieldsetsEpoch = epoch + } + } + cn.preparedFieldsets[name] = version + cn.preparedFieldsetsMu.Unlock() +} + +// UnmarkFieldsetPrepared forgets that the named HIMPORT fieldset was prepared +// on this connection, forcing a replay before the next HIMPORT SET using it. +func (cn *Conn) UnmarkFieldsetPrepared(name string) { + cn.preparedFieldsetsMu.Lock() + delete(cn.preparedFieldsets, name) + cn.preparedFieldsetsMu.Unlock() +} + +// HasPreparedFieldsets reports whether any HIMPORT fieldset is prepared on +// this connection's current server session. +func (cn *Conn) HasPreparedFieldsets() bool { + cn.preparedFieldsetsMu.Lock() + n := len(cn.preparedFieldsets) + cn.preparedFieldsetsMu.Unlock() + return n > 0 +} + +// PreparedFieldsetNames returns the names of the HIMPORT fieldsets prepared +// on this connection's current server session. +func (cn *Conn) PreparedFieldsetNames() []string { + cn.preparedFieldsetsMu.Lock() + names := make([]string, 0, len(cn.preparedFieldsets)) + for name := range cn.preparedFieldsets { + names = append(names, name) + } + cn.preparedFieldsetsMu.Unlock() + return names +} + +// FieldsetEpoch returns the discard-all epoch this connection's prepared +// fieldsets belong to (0 when none were ever prepared on the session). +func (cn *Conn) FieldsetEpoch() uint64 { + cn.preparedFieldsetsMu.Lock() + epoch := cn.preparedFieldsetsEpoch + cn.preparedFieldsetsMu.Unlock() + return epoch +} + +// ClearPreparedFieldsets forgets all HIMPORT fieldsets prepared on this +// connection and records the discard-all epoch the wipe corresponds to. +func (cn *Conn) ClearPreparedFieldsets(epoch uint64) { + cn.preparedFieldsetsMu.Lock() + cn.preparedFieldsets = nil + cn.preparedFieldsetsEpoch = epoch + cn.preparedFieldsetsMu.Unlock() } // GetNetConn safely returns the current network connection using atomic load (lock-free). @@ -694,6 +815,10 @@ func (cn *Conn) SetNetConnAndInitConn(ctx context.Context, netConn net.Conn) err return fmt.Errorf("cannot initialize connection from state %s: %w", finalState, err) } + if cn.onCscReinit != nil { + cn.onCscReinit() + } + // Replace the underlying connection cn.SetNetConn(netConn) @@ -774,8 +899,13 @@ func (cn *Conn) MarkQueuedForHandoff() error { // Already unusable - this is fine, keep the new handoff state return nil } - // Restore the original state if transition fails for other reasons - cn.handoffStateAtomic.Store(currentState) + // Restore the original handoff state only if nothing else changed it + // since our CAS above. A concurrent handoff worker may have completed + // the handoff and run ClearHandoffState in this window; a plain Store + // would clobber that, resurrecting ShouldHandoff=true and wedging the + // connection so it can never be acquired again. The CAS leaves the + // worker's state intact when it has taken over. + cn.handoffStateAtomic.CompareAndSwap(newState, currentState) return fmt.Errorf("failed to mark connection as unusable: %w", err) } return nil @@ -871,6 +1001,18 @@ func (cn *Conn) PeekReplyTypeSafe() (byte, error) { return cn.rd.PeekReplyType() } +// PeekReplyTypeForCheck peeks at the reply type while holding readerMu, so it is +// safe against a concurrent SetNetConn resetting the reader during handoff. +// Unlike PeekReplyTypeSafe it does not require the data to already be buffered: +// the pool health check calls it after connCheck reports unexpected socket data, +// and connCheck only MSG_PEEKs, so the byte still has to be pulled from the +// socket into the reader here. +func (cn *Conn) PeekReplyTypeForCheck() (byte, error) { + cn.readerMu.RLock() + defer cn.readerMu.RUnlock() + return cn.rd.PeekReplyType() +} + func (cn *Conn) Write(b []byte) (int, error) { // Lock-free netConn access for better performance if netConn := cn.getNetConn(); netConn != nil { @@ -907,6 +1049,29 @@ func (cn *Conn) WithReader( return fn(cn.rd) } +// WithReaderHardDeadline runs fn under a HARD read deadline of now+timeout, +// bypassing getEffectiveReadTimeout so a relaxed maintenance timeout can't extend +// it (used by the CSC drainer). Takes no context: an expired cycle ctx must not +// become the socket deadline, or the read surfaces context.DeadlineExceeded, which +// isBadConn treats as fatal. +func (cn *Conn) WithReaderHardDeadline( + timeout time.Duration, fn func(rd *proto.Reader) error, +) (err error) { + netConn := cn.getNetConn() + if netConn == nil { + return errConnectionNotAvailable + } + if err := netConn.SetReadDeadline(time.Now().Add(timeout)); err != nil { + return err + } + defer func() { + if clearErr := netConn.SetReadDeadline(time.Time{}); clearErr != nil { + err = clearErr + } + }() + return fn(cn.rd) +} + func (cn *Conn) WithWriter( ctx context.Context, timeout time.Duration, fn func(wr *proto.Writer) error, ) error { @@ -944,17 +1109,29 @@ func (cn *Conn) IsClosed() bool { } func (cn *Conn) Close() error { - if cn.IsClosed() { - return nil + for { + state := cn.stateMachine.GetState() + if state == StateClosed { + return nil + } + if cn.stateMachine.TryTransitionFast(state, StateClosed) { + // TryTransitionFast deliberately skips waiter notification; Close + // still needs to wake any goroutine waiting on initialization. + cn.stateMachine.notifyWaiters() + break + } } - // Transition to CLOSED state - cn.stateMachine.Transition(StateClosed) if cn.onClose != nil { // ignore error _ = cn.onClose() cn.onClose = nil } + if cn.onCscClose != nil { + // ignore error + _ = cn.onCscClose() + cn.onCscClose = nil + } // Lock-free netConn access for better performance if netConn := cn.getNetConn(); netConn != nil { @@ -974,6 +1151,56 @@ func (cn *Conn) MaybeHasData() bool { return false } +// CheckForData reports whether the socket has data ready and surfaces a +// detected closed or failed socket. +func (cn *Conn) CheckForData() (bool, error) { + if netConn := cn.getNetConn(); netConn != nil { + return checkForData(netConn) + } + return false, nil +} + +// MarkCscReadPending requests one conservative CSC drain after a command read +// when the transport may retain data that MaybeHasData cannot observe. +func (cn *Conn) MarkCscReadPending() { + netConn := cn.getNetConn() + if netConn == nil { + return + } + if needsCscReadProbe(netConn) { + cn.cscReadPending.Store(true) + } +} + +// TakeCscReadPending consumes the post-command conservative-drain request. +func (cn *Conn) TakeCscReadPending() bool { + return cn.cscReadPending.Swap(false) +} + +// TakeCscPeriodicReadPending schedules a throttled conservative read for +// transports with no readiness mechanism. It returns true at most once per +// interval, including when several drainer passes race. +func (cn *Conn) TakeCscPeriodicReadPending(interval time.Duration) bool { + netConn := cn.getNetConn() + if netConn == nil || interval <= 0 || !needsCscPeriodicProbe(netConn) { + return false + } + + now := time.Since(cn.createdAt).Nanoseconds() + if now <= 0 { + now = 1 + } + for { + last := cn.lastCscPeriodicProbeNs.Load() + if last != 0 && now >= last && now-last < int64(interval) { + return false + } + if cn.lastCscPeriodicProbeNs.CompareAndSwap(last, now) { + return true + } + } +} + // deadline computes the effective deadline time based on context and timeout. // It updates the usedAt timestamp to now. // Uses cached time to avoid expensive syscall (max 50ms staleness is acceptable for deadline calculation). diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check.go b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check.go index 9e83dd83..333c34ba 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check.go @@ -19,10 +19,19 @@ func connCheck(conn net.Conn) error { // Reset previous timeout. _ = conn.SetDeadline(time.Time{}) + // Health checks deliberately inspect only the outer connection. Unwrapping a + // buffered transport such as crypto/tls.Conn can reveal an encrypted + // post-handshake record and make isHealthyConn call PeekReplyType on the TLS + // stream. With the deadline cleared above, TLS may consume that control record + // and then wait forever for application data. sysConn, ok := conn.(syscall.Conn) if !ok { return nil } + return checkSyscallConn(sysConn) +} + +func checkSyscallConn(sysConn syscall.Conn) error { rawConn, err := sysConn.SyscallConn() if err != nil { return err @@ -53,7 +62,62 @@ func connCheck(conn net.Conn) error { return sysErr } +// underlyingSyscallConn unwraps connections that expose their transport through +// NetConn (notably crypto/tls.Conn). Limit the walk so a broken wrapper cannot +// loop forever. +func underlyingSyscallConn(conn net.Conn) (syscall.Conn, bool) { + for range 8 { + if sysConn, ok := conn.(syscall.Conn); ok { + return sysConn, true + } + unwrapper, ok := conn.(interface{ NetConn() net.Conn }) + if !ok { + return nil, false + } + conn = unwrapper.NetConn() + if conn == nil { + return nil, false + } + } + return nil, false +} + // maybeHasData checks if there is data in the socket without consuming it func maybeHasData(conn net.Conn) bool { - return connCheck(conn) == errUnexpectedRead + hasData, _ := checkForData(conn) + return hasData +} + +func checkForData(conn net.Conn) (bool, error) { + // Unlike the general health check, CSC only uses this as a readiness hint + // before a bounded read, so unwrapping TLS is safe and avoids blocking. + _ = conn.SetDeadline(time.Time{}) + sysConn, ok := underlyingSyscallConn(conn) + if !ok { + return false, nil + } + switch err := checkSyscallConn(sysConn); err { + case nil: + return false, nil + case errUnexpectedRead: + return true, nil + default: + return false, err + } +} + +// needsCscReadProbe reports whether a command read may leave data hidden from +// maybeHasData. On Unix a direct syscall.Conn has no intermediate buffering; +// TLS and opaque wrappers need one bounded post-command probe. +func needsCscReadProbe(conn net.Conn) bool { + _, direct := conn.(syscall.Conn) + return !direct +} + +// needsCscPeriodicProbe reports whether the platform can inspect the transport +// at all. Opaque wrappers get a throttled bounded fallback so invalidations that +// arrive after the post-command probe are still eventually consumed. +func needsCscPeriodicProbe(conn net.Conn) bool { + _, ok := underlyingSyscallConn(conn) + return !ok } diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check_dummy.go b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check_dummy.go index f971d94c..94770661 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check_dummy.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/conn_check_dummy.go @@ -14,7 +14,21 @@ func connCheck(_ net.Conn) error { return nil } -// since we can't check for data on the socket, we just assume there is some +// There is no portable non-consuming readiness check on this platform. +// Returning true would force every idle CSC connection through a timed read on +// every drainer tick. The CSC drainer uses needsCscPeriodicProbe instead. func maybeHasData(_ net.Conn) bool { + return false +} + +func checkForData(_ net.Conn) (bool, error) { + return false, nil +} + +func needsCscReadProbe(_ net.Conn) bool { + return true +} + +func needsCscPeriodicProbe(_ net.Conn) bool { return true } diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pool.go b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pool.go index 8f648ffb..b79c2bac 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pool.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pool.go @@ -72,6 +72,11 @@ var ( // errConnNotPooled is returned when trying to return a non-pooled connection to the pool. errConnNotPooled = errors.New("connection not pooled") + + // errConnEvictedIdle is passed to OnRemove hooks when a pooled connection is evicted on + // Put because the idle pool is already at MaxIdleConns. + errConnEvictedIdle = errors.New("connection evicted: idle pool at capacity") + // metricCallbackMu protects all global metric callback functions for thread-safe access. metricCallbackMu sync.RWMutex @@ -304,6 +309,9 @@ func getMetricPendingRequestsCallback() func(ctx context.Context, delta int, cn } // Stats contains pool state information and accumulated stats. +// +// TODO(cxl): the uint32/int64 fields below will be changed to atomic value +// types (atomic.Uint32/atomic.Int64) in v10, which is a breaking API change. type Stats struct { Hits uint32 // number of times free connection was found in the pool Misses uint32 // number of times free connection was NOT found in the pool @@ -318,6 +326,11 @@ type Stats struct { PendingRequests uint32 // number of pending requests waiting for a connection PubSubStats PubSubStats + + // PipelineStats holds the stats of the separate pipeline connection pool + // when one is configured (PipelineReadBufferSize/PipelineWriteBufferSize). + // nil when pipelines share the main pool. + PipelineStats *Stats } type ConnRetirer interface { @@ -394,7 +407,7 @@ type lastDialErrorWrap struct { type ConnPool struct { cfg *Options - dialErrorsNum uint32 // atomic + dialErrorsNum atomic.Uint32 lastDialError atomic.Value dialsInProgress chan struct{} @@ -415,11 +428,23 @@ type ConnPool struct { stats Stats waitDurationNs atomic.Int64 - _closed uint32 // atomic + _closed atomic.Uint32 - // Pool hooks manager for flexible connection processing - // Using atomic.Pointer for lock-free reads in hot paths (Get/Put) + // Pool hooks manager. atomic.Pointer keeps hot-path reads (Get/Put) + // lock-free; hookMu serializes Add/RemovePoolHook's read-clone-store so + // concurrent mutators (e.g. maintnotifications and CSC) can't lose an update. hookManager atomic.Pointer[PoolHookManager] + hookMu sync.Mutex + + // drainMu/drainDone coordinate the CSC drainer's temporary idle-connection + // claim with Get. The normal semaphore retains its PoolSize capacity (and + // therefore the established MaxActiveConns/ErrPoolExhausted behavior); a Get + // that finds the idle list empty only because the drainer borrowed a conn + // waits for that short claim to finish instead of opening an overflow conn. + drainMu sync.Mutex + drainDone chan struct{} + drainBorrowed int + drainGeneration atomic.Uint64 } var _ Pooler = (*ConnPool)(nil) @@ -453,7 +478,10 @@ func (p *ConnPool) initializeHooks() { // AddPoolHook adds a pool hook to the pool. func (p *ConnPool) AddPoolHook(hook PoolHook) { - // Lock-free read of current manager + // Serialize so a concurrent Add/Remove can't clobber this change. + p.hookMu.Lock() + defer p.hookMu.Unlock() + manager := p.hookManager.Load() if manager == nil { p.initializeHooks() @@ -464,12 +492,22 @@ func (p *ConnPool) AddPoolHook(hook PoolHook) { newManager := manager.Clone() newManager.AddHook(hook) - // Atomically swap to new manager + // Atomically swap to new manager (hot-path readers load lock-free) p.hookManager.Store(newManager) } +// SupportsPoolHooks reports that AddPoolHook and RemovePoolHook are functional. +// Pooler adapters with no-op hook methods intentionally do not expose this +// optional capability. +func (p *ConnPool) SupportsPoolHooks() bool { + return true +} + // RemovePoolHook removes a pool hook from the pool. func (p *ConnPool) RemovePoolHook(hook PoolHook) { + p.hookMu.Lock() + defer p.hookMu.Unlock() + manager := p.hookManager.Load() if manager != nil { // Create new manager with removed hook @@ -651,7 +689,7 @@ func (p *ConnPool) dialConn(ctx context.Context, pooled bool) (*Conn, error) { return nil, ErrClosed } - if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.cfg.PoolSize) { + if p.dialErrorsNum.Load() >= uint32(p.cfg.PoolSize) { return nil, p.getLastDialError() } @@ -724,7 +762,7 @@ func (p *ConnPool) dialConn(ctx context.Context, pooled bool) (*Conn, error) { internal.Logger.Printf(ctx, "redis: connection pool: failed to dial after %d attempts: %v", attempt, lastErr) // All retries failed - handle error tracking p.setLastDialError(lastErr) - if atomic.AddUint32(&p.dialErrorsNum, 1) == uint32(p.cfg.PoolSize) { + if p.dialErrorsNum.Add(1) == uint32(p.cfg.PoolSize) { go p.tryDial() } return nil, lastErr @@ -789,7 +827,7 @@ func (p *ConnPool) tryDial() { continue } - atomic.StoreUint32(&p.dialErrorsNum, 0) + p.dialErrorsNum.Store(0) _ = conn.Close() return } @@ -835,28 +873,30 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) { cb(ctx, -1, nil, poolName) } } + if err == ErrPoolTimeout { + atomic.AddUint32(&p.stats.Timeouts, 1) + if cb := getMetricConnectionTimeoutCallback(); cb != nil { + cb(ctx, nil, "pool") + } + if cb := GetMetricErrorCallback(); cb != nil { + cb(ctx, "POOL_TIMEOUT", nil, "POOL_TIMEOUT", true, 0) + } + } }() - // Track wait time - only call time.Now() if callback is registered + // PoolTimeout is one budget for both the pool turn and a drainer handoff. + poolDeadline := time.Now().Add(p.cfg.PoolTimeout) + + // Connection wait time measures only semaphore acquisition. var waitStart time.Time + var waitDuration time.Duration waitTimeCallback := getMetricConnectionWaitTimeCallback() if waitTimeCallback != nil { waitStart = time.Now() } if err = p.waitTurn(ctx); err != nil { - // Record timeout if applicable - if err == ErrPoolTimeout { - if cb := getMetricConnectionTimeoutCallback(); cb != nil { - cb(ctx, nil, "pool") - } - // Record general error metric for pool timeout - if cb := GetMetricErrorCallback(); cb != nil { - cb(ctx, "POOL_TIMEOUT", nil, "POOL_TIMEOUT", true, 0) - } - } return nil, err } - var waitDuration time.Duration if waitTimeCallback != nil { waitDuration = time.Since(waitStart) } @@ -867,6 +907,8 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) { // Lock-free atomic read - no mutex overhead! hookManager := p.hookManager.Load() +retryIdle: + drainGeneration := p.drainGeneration.Load() for attempts := 0; attempts < getAttempts; attempts++ { p.connsMu.Lock() @@ -937,6 +979,21 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) { return cn, nil } + // If the CSC drainer removed the only idle connection during this scan, + // wait for that bounded maintenance claim and retry. The generation closes + // the race where the drainer returns the connection between popIdle and this + // check. Normal MaxActiveConns exhaustion still proceeds to newConn and + // returns ErrPoolExhausted immediately, preserving the existing contract. + if done, retry := p.drainerWaitState(drainGeneration); done != nil { + if err = p.waitForDrainer(ctx, done, poolDeadline); err != nil { + p.freeTurn() + return nil, err + } + goto retryIdle + } else if retry { + goto retryIdle + } + atomic.AddUint32(&p.stats.Misses, 1) var newcn *Conn @@ -964,7 +1021,7 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) { // causing IsInited()=true. This means _getConn() in redis.go will take the // early return path and never reach its create time recording. // When hookManager is nil, _getConn() handles both initialization and create time recording. - if dialStartNs := newcn.GetDialStartNs(); dialStartNs > 0 { + if dialStartNs := newcn.GetDialStartNs(); newcn.IsInited() && dialStartNs > 0 { if cb := GetMetricConnectionCreateTimeCallback(); cb != nil { duration := time.Duration(time.Now().UnixNano() - dialStartNs) cb(ctx, duration, newcn) @@ -1120,23 +1177,83 @@ func (p *ConnPool) waitTurn(ctx context.Context) error { // Slow path: need to wait start := time.Now() err := p.semaphore.Acquire(ctx, p.cfg.PoolTimeout, ErrPoolTimeout) - - switch err { - case nil: - // Successfully acquired after waiting - p.waitDurationNs.Add(time.Now().UnixNano() - start.UnixNano()) - atomic.AddUint32(&p.stats.WaitCount, 1) - case ErrPoolTimeout: - atomic.AddUint32(&p.stats.Timeouts, 1) + if err != nil { + return err } - return err + p.waitDurationNs.Add(time.Now().UnixNano() - start.UnixNano()) + atomic.AddUint32(&p.stats.WaitCount, 1) + return nil } func (p *ConnPool) freeTurn() { p.semaphore.Release() } +func (p *ConnPool) beginDrainerBorrow() { + p.drainMu.Lock() + if p.drainBorrowed == 0 { + p.drainDone = make(chan struct{}) + } + p.drainBorrowed++ + p.drainMu.Unlock() +} + +func (p *ConnPool) endDrainerBorrow() { + p.drainMu.Lock() + p.drainBorrowed-- + if p.drainBorrowed == 0 { + close(p.drainDone) + p.drainDone = nil + p.drainGeneration.Add(1) + } + p.drainMu.Unlock() +} + +// drainerWaitState returns the current drain epoch's completion channel. If no +// drain is active, retry reports whether an epoch completed during the caller's +// idle scan and the idle list therefore needs to be checked again. +func (p *ConnPool) drainerWaitState(generation uint64) (done <-chan struct{}, retry bool) { + p.drainMu.Lock() + defer p.drainMu.Unlock() + if p.drainBorrowed > 0 { + return p.drainDone, false + } + return nil, p.drainGeneration.Load() != generation +} + +func (p *ConnPool) waitForDrainer( + ctx context.Context, done <-chan struct{}, poolDeadline time.Time, +) error { + if err := ctx.Err(); err != nil { + return err + } + select { + case <-done: + return nil + default: + } + remaining := time.Until(poolDeadline) + if remaining <= 0 { + return ErrPoolTimeout + } + timer := time.NewTimer(remaining) + defer timer.Stop() + + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + // Prefer a caller cancellation that raced with the pool timeout. + if err := ctx.Err(); err != nil { + return err + } + return ErrPoolTimeout + } +} + func (p *ConnPool) popIdle() (*Conn, error) { if p.closed() { return nil, ErrClosed @@ -1290,6 +1407,9 @@ func (p *ConnPool) putConn(ctx context.Context, cn *Conn, freeTurn bool) { // expected state, don't log it case StateClosed: internal.Logger.Printf(ctx, "Unexpected conn[%d] state changed by hook to %v, closing it", cn.GetID(), currentState) + if hookManager != nil { + hookManager.ProcessOnRemove(ctx, cn, errHookRequestedRemoval) + } shouldCloseConn = true removedFromPool = p.removeConnWithLock(cn) default: @@ -1357,6 +1477,9 @@ func (p *ConnPool) putConn(ctx context.Context, cn *Conn, freeTurn bool) { } } else { shouldCloseConn = true + if hookManager != nil { + hookManager.ProcessOnRemove(ctx, cn, errConnEvictedIdle) + } removedFromPool = p.removeConnWithLock(cn) // Only emit if we actually removed it from the map (not already taken by Close()). @@ -1548,6 +1671,13 @@ func (p *ConnPool) IdleLen() int { return int(n) } +// Name returns the pool's configured name, which is stamped on every +// connection it creates (Conn.PoolName). Callers holding a Pooler can type +// assert to interface{ Name() string } to find which pool owns a connection — +// used by maintnotifications to route a handoff to the hook that owns the +// conn's pool rather than always the primary one. +func (p *ConnPool) Name() string { return p.cfg.Name } + // Size returns the maximum pool size (capacity). // // This is used by the streaming credentials manager to size the re-auth worker pool, @@ -1573,7 +1703,7 @@ func (p *ConnPool) Stats() *Stats { } func (p *ConnPool) closed() bool { - return atomic.LoadUint32(&p._closed) == 1 + return p._closed.Load() == 1 } func (p *ConnPool) RetireConns(ctx context.Context, conns []*Conn, reason string) { @@ -1652,8 +1782,142 @@ func (p *ConnPool) Filter(fn func(*Conn) bool) error { return firstErr } +type drainConn struct { + conn *Conn + idleIndex int +} + +// DrainState carries the cross-pass round bookkeeping for the CSC drainer. +// It is owned by one drainer goroutine, so no synchronization is needed. +type DrainState struct { + // round contains only initialized idle connections. Entries are processed + // from the end so their snapshot indexes remain stable as connections are + // removed and returned. Connections that go idle mid-round are deferred. + round []drainConn + next int +} + +// DrainIdleConns runs one pass of the CSC invalidation drainer over the current +// round, holding AT MOST ONE connection and its pool turn at a time (ctx is the +// per-cycle deadline). A round = idle conn ids snapshotted at start; mid-round +// arrivals are deferred. Each member is drainerPop'd and drained by fn, or — if no +// longer a claimable idle conn — reconciled (marked visited) so it can't hang the +// round. The drainer yields when no turn is immediately available, giving command +// traffic priority. Handles at least one member before honoring ctx (so a tiny +// DrainInterval can't stall it). No-ops if the pool is closed. +func (p *ConnPool) DrainIdleConns(ctx context.Context, st *DrainState, fn func(cn *Conn) error) { + if st == nil || fn == nil || p.closed() { + return + } + + if st.round == nil { + st.round = p.idleConnsSnapshot() + st.next = len(st.round) + if len(st.round) == 0 { + st.round = nil + return + } + } + + handled := 0 + for st.next > 0 { + // Min-progress: handle at least one member — drained OR reconciled — before + // honoring the per-cycle deadline, so a pass does a bounded amount of work + // while ignoring an expired ctx. A deadline-truncated round resumes on the + // next pass. + if handled > 0 && ctx.Err() != nil { + return + } + + // Account for the borrowed connection exactly like Get. Without a turn, + // a concurrent Get can observe the temporarily-empty idle pool and either + // exceed PoolSize or fail at MaxActiveConns. Maintenance never waits for a + // turn, so command traffic wins under contention. + if !p.semaphore.TryAcquire() { + return + } + st.next-- + cn := p.drainerPop(ctx, st.round[st.next]) + if cn == nil { + p.freeTurn() + // Reconcile: not a claimable idle member right now (closed, in use, + // unusable, or moved in idleConns by concurrent traffic). Covered by + // the command-path drain and/or the next round. + handled++ + continue + } + + func() { + defer p.endDrainerBorrow() + if err := fn(cn); err != nil { + // Fatal drain error (read/protocol/connection). + p.removeConnInternal(ctx, cn, err, true) + } else { + // Normal return: runs OnPut (queues any maintenance handoff). + p.putConn(ctx, cn, true) + } + }() + handled++ + } + + // Every member handled — round complete; snapshot a fresh round next pass. + st.round = nil + st.next = 0 +} + +// idleConnsSnapshot returns initialized idle connections and their current +// indexes. StateCreated MinIdleConns are intentionally excluded. +func (p *ConnPool) idleConnsSnapshot() []drainConn { + p.connsMu.Lock() + defer p.connsMu.Unlock() + if len(p.idleConns) == 0 { + return nil + } + round := make([]drainConn, 0, len(p.idleConns)) + for idx, cn := range p.idleConns { + if cn.stateMachine.GetState() == StateIdle { + round = append(round, drainConn{conn: cn, idleIndex: idx}) + } + } + return round +} + +// drainerPop claims a snapshotted connection (strict IDLE->IN_USE) and removes +// it from idleConns in O(1). Entries are processed in reverse index order, so +// swap removal cannot move an unprocessed round member. If concurrent pool +// traffic changed the slot, the member is deferred to the next round. +func (p *ConnPool) drainerPop(ctx context.Context, member drainConn) *Conn { + p.connsMu.Lock() + defer p.connsMu.Unlock() + if p.closed() { + return nil + } + idx := member.idleIndex + if idx < 0 || idx >= len(p.idleConns) || p.idleConns[idx] != member.conn { + return nil + } + cn := member.conn + if !cn.stateMachine.TryTransitionFast(StateIdle, StateInUse) { + return nil + } + p.beginDrainerBorrow() + last := len(p.idleConns) - 1 + p.idleConns[idx] = p.idleConns[last] + p.idleConns[last] = nil + p.idleConns = p.idleConns[:last] + p.idleConnsLen.Add(-1) + if cb := getMetricConnectionStateChangeCallback(); cb != nil { + cb(ctx, cn, MetricStateIdle, MetricStateUsed) + } + if cb := getMetricConnectionCountCallback(); cb != nil { + cb(ctx, -1, cn, "idle", false) + cb(ctx, 1, cn, "used", false) + } + return cn +} + func (p *ConnPool) Close() error { - if !atomic.CompareAndSwapUint32(&p._closed, 0, 1) { + if !p._closed.CompareAndSwap(0, 1) { return ErrClosed } @@ -1735,8 +1999,11 @@ func (p *ConnPool) isHealthyConn(cn *Conn, nowNs int64) bool { if err := connCheck(cn.getNetConn()); err != nil { // If there's unexpected data, it might be push notifications (RESP3) if p.cfg.PushNotificationsEnabled && err == errUnexpectedRead { - // Peek at the reply type to check if it's a push notification - if replyType, err := cn.rd.PeekReplyType(); err == nil && replyType == proto.RespPush { + // Peek at the reply type to check if it's a push notification. + // Use the readerMu-guarded peek: a concurrent handoff may be + // resetting cn.rd via SetNetConn on a connection popped by Get + // before the OnGet state check rejects it. + if replyType, err := cn.PeekReplyTypeForCheck(); err == nil && replyType == proto.RespPush { // For RESP3 connections with push notifications, we allow some buffered data // The client will process these notifications before using the connection internal.Logger.Printf( diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pool_sticky.go b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pool_sticky.go index 6763299e..e2206c82 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pool_sticky.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pool_sticky.go @@ -35,11 +35,16 @@ func (e BadConnError) Unwrap() error { type StickyConnPool struct { pool Pooler - shared int32 // atomic + shared atomic.Int32 - state uint32 // atomic + state atomic.Uint32 ch chan *Conn + // onFirstConn runs once when this sticky pool claims a connection from its + // parent. CSC uses it to revoke cache ownership before the connection leaves + // the parent's background drainer. + onFirstConn func(*Conn) + _badConnError atomic.Value } @@ -53,7 +58,7 @@ func NewStickyConnPool(pool Pooler) *StickyConnPool { ch: make(chan *Conn, 1), } } - atomic.AddInt32(&p.shared, 1) + p.shared.Add(1) return p } @@ -68,13 +73,16 @@ func (p *StickyConnPool) CloseConn(ctx context.Context, cn *Conn, reason string, func (p *StickyConnPool) Get(ctx context.Context) (*Conn, error) { // In worst case this races with Close which is not a very common operation. for i := 0; i < 1000; i++ { - switch atomic.LoadUint32(&p.state) { + switch p.state.Load() { case stateDefault: cn, err := p.pool.Get(ctx) if err != nil { return nil, err } - if atomic.CompareAndSwapUint32(&p.state, stateDefault, stateInited) { + if p.state.CompareAndSwap(stateDefault, stateInited) { + if p.onFirstConn != nil { + p.onFirstConn(cn) + } return cn, nil } p.pool.Remove(ctx, cn, ErrClosed) @@ -96,12 +104,26 @@ func (p *StickyConnPool) Get(ctx context.Context) (*Conn, error) { return nil, fmt.Errorf("redis: StickyConnPool.Get: infinite loop") } +// SetOnFirstConn configures a callback that runs when the sticky pool first +// claims a parent connection. It must be called before the pool is used. +func (p *StickyConnPool) SetOnFirstConn(fn func(*Conn)) { + p.onFirstConn = fn +} + func (p *StickyConnPool) Put(ctx context.Context, cn *Conn) { defer func() { if recover() != nil { p.freeConn(ctx, cn) } }() + // A connection marked for removal on release (it may hold unread + // replies) must not be served to the next Get: record it as a bad + // connection — exactly like Remove — so Get refuses and the underlying + // connection is removed from the parent pool when the sticky pool + // unwinds (the parent's Put honors the same mark). + if reason := cn.CloseOnPutReason(); reason != "" { + p._badConnError.Store(BadConnError{wrapped: errors.New(reason)}) + } p.ch <- cn } @@ -130,16 +152,16 @@ func (p *StickyConnPool) RemoveWithoutTurn(ctx context.Context, cn *Conn, reason } func (p *StickyConnPool) Close() error { - if shared := atomic.AddInt32(&p.shared, -1); shared > 0 { + if shared := p.shared.Add(-1); shared > 0 { return nil } for i := 0; i < 1000; i++ { - state := atomic.LoadUint32(&p.state) + state := p.state.Load() if state == stateClosed { return ErrClosed } - if atomic.CompareAndSwapUint32(&p.state, state, stateClosed) { + if p.state.CompareAndSwap(state, stateClosed) { close(p.ch) cn, ok := <-p.ch if ok { @@ -168,8 +190,8 @@ func (p *StickyConnPool) Reset(ctx context.Context) error { return errors.New("redis: StickyConnPool does not have a Conn") } - if !atomic.CompareAndSwapUint32(&p.state, stateInited, stateDefault) { - state := atomic.LoadUint32(&p.state) + if !p.state.CompareAndSwap(stateInited, stateDefault) { + state := p.state.Load() return fmt.Errorf("redis: invalid StickyConnPool state: %d", state) } @@ -186,7 +208,7 @@ func (p *StickyConnPool) badConnError() error { } func (p *StickyConnPool) Len() int { - switch atomic.LoadUint32(&p.state) { + switch p.state.Load() { case stateDefault: return 0 case stateInited: diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pubsub.go b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pubsub.go index 8cfa8678..34c329ff 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pubsub.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/pool/pubsub.go @@ -7,6 +7,10 @@ import ( "sync/atomic" ) +// PubSubStats contains pub/sub connection pool stats. +// +// TODO(cxl): the uint32 fields below will be changed to atomic.Uint32 in v10, +// which is a breaking API change. type PubSubStats struct { Created uint32 Untracked uint32 diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/proto/reader.go b/backend/vendor/github.com/redis/go-redis/v9/internal/proto/reader.go index 33b027f7..28e1a781 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/proto/reader.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/proto/reader.go @@ -76,6 +76,11 @@ func (r *Reader) Buffered() int { return r.rd.Buffered() } +// Size returns the size of the underlying read buffer. +func (r *Reader) Size() int { + return r.rd.Size() +} + func (r *Reader) Peek(n int) ([]byte, error) { return r.rd.Peek(n) } @@ -100,18 +105,30 @@ func (r *Reader) PeekReplyType() (byte, error) { return b[0], nil } +// MinRESP3ReadBufferSize is the minimum buffer size used when RESP3 push +// notifications must be inspected without consuming them. +const MinRESP3ReadBufferSize = 128 + +// ErrPushNotificationNameTooLong is returned when the push header does not fit +// in the bounded peek window. Callers should consume the frame with ReadReply. +var ErrPushNotificationNameTooLong = errors.New("redis: push notification name exceeds peek window") + // PeekPushNotificationName returns the notification name of the next RESP3 // push frame without consuming it. The caller is expected to have already // verified that the next reply is a push notification (e.g. via PeekReplyType // returning RespPush). // -// To identify the name the method may block briefly reading more bytes from -// the underlying connection. That is safe: once the push marker '>' has been -// observed, the server is committed to sending the rest of the frame, so -// fetching the next few header bytes does not race with anything the caller -// could be waiting on. Blocking is preferred to a truncated peek, which would -// silently misidentify the notification and cause the caller's ReadReply to -// consume (and drop) the frame; see issue #3839. +// To identify the name the method may block reading more bytes from the +// underlying connection, but only ever waits for one byte beyond the valid +// frame prefix it has already seen. That byte is guaranteed to arrive: an +// incomplete prefix means the server is still committed to sending the rest +// of the frame. Demanding any fixed amount instead can deadlock — a complete +// frame such as a subscribe confirmation for a short channel name can be +// smaller than the fixed window, and once it is buffered the server has +// nothing more to send (issue #3935). Blocking for in-flight bytes is +// preferred to a truncated peek, which would silently misidentify the +// notification and cause the caller's ReadReply to consume (and drop) the +// frame; see issue #3839. func (r *Reader) PeekPushNotificationName() (string, error) { c, err := r.rd.Peek(1) if err != nil { @@ -121,16 +138,18 @@ func (r *Reader) PeekPushNotificationName() (string, error) { return "", fmt.Errorf("redis: can't peek push notification name, next reply is not a push notification") } - // Start with a peek window that covers every Redis-defined notification - // header (MOVING, MIGRATING, FAILED_OVER, message, pmessage, smessage, - // subscribe, unsubscribe, ...). If a longer name is encountered, grow - // the window up to maxPushHeaderPeek before giving up. - const initialPeek = 36 const maxPushHeaderPeek = 4096 - peekSize := initialPeek for { - buf, peekErr := r.rd.Peek(peekSize) + // Parse from what is already buffered; this never blocks. + avail := r.rd.Buffered() + if avail > maxPushHeaderPeek { + avail = maxPushHeaderPeek + } + buf, peekErr := r.rd.Peek(avail) + if peekErr != nil { + return "", peekErr + } name, complete, parseErr := parsePushNotificationName(buf) if parseErr != nil { return "", parseErr @@ -138,17 +157,17 @@ func (r *Reader) PeekPushNotificationName() (string, error) { if complete { return name, nil } - // Parser ran out of bytes. Surface a failed underlying read before - // growing further; otherwise grow the peek window and retry. - if peekErr != nil { - return "", peekErr + if avail >= maxPushHeaderPeek { + return "", ErrPushNotificationNameTooLong } - if peekSize >= maxPushHeaderPeek { - return "", fmt.Errorf("redis: push notification header exceeds %d bytes", maxPushHeaderPeek) - } - peekSize *= 2 - if peekSize > maxPushHeaderPeek { - peekSize = maxPushHeaderPeek + // Valid but incomplete prefix: the rest of the frame is in flight. + // Block for exactly one more byte — the read that delivers it picks + // up whatever else has already arrived — then re-parse. + if _, err := r.rd.Peek(avail + 1); err != nil { + if errors.Is(err, bufio.ErrBufferFull) { + return "", ErrPushNotificationNameTooLong + } + return "", err } } } @@ -440,6 +459,16 @@ func (r *Reader) readMap(line []byte) (map[interface{}]interface{}, error) { if err != nil { return nil, err } + + // Reject unhashable keys (arrays/maps) before they are used as a map + // key, which would otherwise panic. This check must run before the + // value is read so it also guards the Nil and RedisError paths below, + // which write the key into the map and continue. + switch k.(type) { + case []interface{}, map[interface{}]interface{}: + return nil, fmt.Errorf("redis: RESP3 map key must be a scalar type, got %T", k) + } + v, err := r.ReadReply() if err != nil { if err == Nil { @@ -452,6 +481,7 @@ func (r *Reader) readMap(line []byte) (map[interface{}]interface{}, error) { } return nil, err } + m[k] = v } return m, nil @@ -591,10 +621,33 @@ func (r *Reader) ReadStringInto(buf []byte) (int, error) { // bufio.Reader.Read first drains its internal buffer, then for // remaining data larger than its buffer size reads directly from the // underlying reader (socket) — effectively zero-copy. + // + // Fast path: when the caller VISIBLY hands over room for the trailing + // CRLF too (len(buf) >= n+2), read the payload and the CRLF in a + // single io.ReadFull. For large values this is one direct socket read + // instead of a big read followed by a tiny separate Discard(2) read, + // which is what makes GetToBuffer beat a regular Get (no payload + // allocation and the same number of reads). The 2 trailing bytes land + // past the returned length and are ignored. + // + // The gate is on len, NOT cap: a sub-slice of a larger buffer (e.g. + // packed segments big[i*slot:(i+1)*slot]) exposes trailing capacity + // that belongs to the caller's NEXT segment — writing the CRLF there + // would silently corrupt caller-owned memory outside the slice they + // passed. Callers who want the fast path pass len == payload+2 (the + // returned length is still the payload length). + if len(buf) >= n+2 { + full := buf[:n+2] + if _, err := io.ReadFull(r.rd, full); err != nil { + return 0, err + } + return n, nil + } + // Slow path: buffer is exactly large enough for the payload only, so + // read the payload into it and discard the CRLF separately. if _, err := io.ReadFull(r.rd, buf[:n]); err != nil { return 0, err } - // Discard trailing \r\n. if _, err := r.rd.Discard(2); err != nil { return 0, err } @@ -738,7 +791,15 @@ func (r *Reader) Discard(line []byte) (err error) { } n, err := replyLen(line) - if err != nil && err != Nil { + if err != nil { + if err == Nil { + // A nil reply ($-1, =-1, !-1, *-1, %-1) carries no payload; the + // header line was already consumed by readLine, so there is + // nothing to discard. Falling through would Discard(n+2)==2 bytes + // that belong to the next reply and desync the stream, matching + // how readRawReplyBuf/readRawReplyWriteTo already treat Nil. + return nil + } return err } @@ -755,8 +816,14 @@ func (r *Reader) Discard(line []byte) (err error) { } return nil case RespMap, RespAttr: - // Read key & value. - for i := 0; i < n*2; i++ { + // Iterate over the n key/value pairs rather than n*2 elements: a count + // above MaxInt/2 makes n*2 overflow to a negative loop bound, which + // would skip the body entirely and return nil, leaving the map bytes in + // the stream for the next reply to consume (a silent desync). + for i := 0; i < n; i++ { + if err = r.DiscardNext(); err != nil { + return err + } if err = r.DiscardNext(); err != nil { return err } @@ -849,10 +916,12 @@ func (r *Reader) readRawReplyBuf(buf []byte) ([]byte, error) { } return buf, err } - for i := 0; i < n*2; i++ { - buf, err = r.readRawReplyBuf(buf) - if err != nil { - return buf, err + for i := 0; i < n; i++ { + for pair := 0; pair < 2; pair++ { + buf, err = r.readRawReplyBuf(buf) + if err != nil { + return buf, err + } } } return buf, nil @@ -867,11 +936,15 @@ func (r *Reader) readRawReplyBuf(buf []byte) ([]byte, error) { } return buf, err } - // Read the attribute key-value pairs - for i := 0; i < n*2; i++ { - buf, err = r.readRawReplyBuf(buf) - if err != nil { - return buf, err + // Read the attribute key-value pairs. Iterate over pairs rather than + // n*2 elements so a count above MaxInt/2 can't overflow int to a + // negative loop bound and skip the body. + for i := 0; i < n; i++ { + for pair := 0; pair < 2; pair++ { + buf, err = r.readRawReplyBuf(buf) + if err != nil { + return buf, err + } } } // Read the command reply that follows the attribute @@ -948,11 +1021,13 @@ func (r *Reader) readRawReplyWriteTo(w io.Writer) (int64, error) { } return written, err } - for i := 0; i < count*2; i++ { - n, err := r.readRawReplyWriteTo(w) - written += n - if err != nil { - return written, err + for i := 0; i < count; i++ { + for pair := 0; pair < 2; pair++ { + n, err := r.readRawReplyWriteTo(w) + written += n + if err != nil { + return written, err + } } } return written, nil @@ -967,12 +1042,16 @@ func (r *Reader) readRawReplyWriteTo(w io.Writer) (int64, error) { } return written, err } - // Read the attribute key-value pairs - for i := 0; i < count*2; i++ { - n, err := r.readRawReplyWriteTo(w) - written += n - if err != nil { - return written, err + // Read the attribute key-value pairs. Iterate over pairs rather than + // count*2 elements so a count above MaxInt/2 can't overflow int to a + // negative loop bound and skip the body. + for i := 0; i < count; i++ { + for pair := 0; pair < 2; pair++ { + n, err := r.readRawReplyWriteTo(w) + written += n + if err != nil { + return written, err + } } } // Read the command reply that follows the attribute diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/proto/writer.go b/backend/vendor/github.com/redis/go-redis/v9/internal/proto/writer.go index 38e66c68..e3eff8b6 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/proto/writer.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/proto/writer.go @@ -118,7 +118,7 @@ func (w *Writer) WriteArg(v interface{}) error { return w.uint(uint64(v)) case *uint8: if v == nil { - return w.string("") + return w.uint(0) } return w.uint(uint64(*v)) case uint16: diff --git a/backend/vendor/github.com/redis/go-redis/v9/internal/semaphore.go b/backend/vendor/github.com/redis/go-redis/v9/internal/semaphore.go index a7f40466..b8d60a7b 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/internal/semaphore.go +++ b/backend/vendor/github.com/redis/go-redis/v9/internal/semaphore.go @@ -14,6 +14,28 @@ var semTimers = sync.Pool{ }, } +// putSemTimer stops a pooled timer and drains a stale fire before reuse, +// portably across both timer-channel semantics (the drain must never block): +// +// - main module on go >= 1.23 (synchronous channels): Stop returns TRUE for +// an expired-but-undelivered fire — the delivery is aborted — and false +// only once the value was actually received. With the sole receiver +// being Acquire's own select, the drain branch is unreachable; the +// select-with-default is a safety net so a future semantics shift cannot +// turn it into a blocking receive (reviewed on #3942). +// - GODEBUG=asynctimerchan=1 (consumer main module on go < 1.23, old +// buffered channels): Stop returns false and the fired value sits in the +// buffer; the drain consumes it so the timer is clean for Reset-reuse. +func putSemTimer(t *time.Timer) { + if !t.Stop() { + select { + case <-t.C: + default: + } + } + semTimers.Put(t) +} + // FastSemaphore is a channel-based semaphore optimized for performance. // It uses a fast path that avoids timer allocation when tokens are available. // The channel is pre-filled with tokens: Acquire = receive, Release = send. @@ -70,19 +92,13 @@ func (s *FastSemaphore) Acquire(ctx context.Context, timeout time.Duration, time // Slow path: need to wait with timeout timer := semTimers.Get().(*time.Timer) - defer semTimers.Put(timer) + defer putSemTimer(timer) timer.Reset(timeout) select { case <-s.tokens: - if !timer.Stop() { - <-timer.C - } return nil case <-ctx.Done(): - if !timer.Stop() { - <-timer.C - } return ctx.Err() case <-timer.C: return timeoutErr @@ -152,42 +168,20 @@ func (s *FIFOSemaphore) TryAcquire() bool { func (s *FIFOSemaphore) Acquire(ctx context.Context, timeout time.Duration, timeoutErr error) error { // No fast path - always use timer to guarantee FIFO timer := semTimers.Get().(*time.Timer) - defer semTimers.Put(timer) + defer putSemTimer(timer) timer.Reset(timeout) select { case <-s.tokens: - if !timer.Stop() { - <-timer.C - } return nil case <-ctx.Done(): - if !timer.Stop() { - <-timer.C - } return ctx.Err() case <-timer.C: return timeoutErr } } -// AcquireBlocking acquires a token, blocking indefinitely until one is available. -func (s *FIFOSemaphore) AcquireBlocking() { - <-s.tokens -} - // Release releases a token back to the semaphore. func (s *FIFOSemaphore) Release() { s.tokens <- struct{}{} } - -// Close closes the semaphore, unblocking all waiting goroutines. -// After close, all Acquire calls will receive a closed channel signal. -func (s *FIFOSemaphore) Close() { - close(s.tokens) -} - -// Len returns the current number of acquired tokens. -func (s *FIFOSemaphore) Len() int32 { - return s.max - int32(len(s.tokens)) -} diff --git a/backend/vendor/github.com/redis/go-redis/v9/iterator.go b/backend/vendor/github.com/redis/go-redis/v9/iterator.go index cd1a8285..a0cf1b35 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/iterator.go +++ b/backend/vendor/github.com/redis/go-redis/v9/iterator.go @@ -46,6 +46,14 @@ func (it *ScanIterator) Next(ctx context.Context) bool { if err != nil { return false } + // Await the fetch before reading page/cursor: on the deferred + // autopipeline face process() only enqueues, and reading the raw + // fields of a not-yet-executed command would spin re-issuing SCANs + // with a stale cursor forever. Err() blocks until executed there and + // is a no-op read everywhere else. + if err := it.cmd.Err(); err != nil { + return false + } it.pos = 1 diff --git a/backend/vendor/github.com/redis/go-redis/v9/json.go b/backend/vendor/github.com/redis/go-redis/v9/json.go index 2bcad0b7..b3878260 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/json.go +++ b/backend/vendor/github.com/redis/go-redis/v9/json.go @@ -96,6 +96,7 @@ func newJSONCmd(ctx context.Context, args ...interface{}) *JSONCmd { } func (cmd *JSONCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -105,6 +106,7 @@ func (cmd *JSONCmd) SetVal(val string) { // Val returns the result of the JSON.GET command as a string. func (cmd *JSONCmd) Val() string { + cmd.await() if len(cmd.val) == 0 && cmd.expanded != nil { val, err := json.Marshal(cmd.expanded) if err != nil { @@ -119,11 +121,13 @@ func (cmd *JSONCmd) Val() string { } func (cmd *JSONCmd) Result() (string, error) { + cmd.await() return cmd.Val(), cmd.Err() } // Expanded returns the result of the JSON.GET command as unmarshalled JSON. func (cmd *JSONCmd) Expanded() (interface{}, error) { + cmd.await() if len(cmd.val) != 0 && cmd.expanded == nil { err := json.Unmarshal([]byte(cmd.val), &cmd.expanded) if err != nil { @@ -136,15 +140,18 @@ func (cmd *JSONCmd) Expanded() (interface{}, error) { func (cmd *JSONCmd) readReply(rd *proto.Reader) error { // nil response from JSON.(M)GET (cmd.baseCmd.err will be "redis: nil") - // This happens when the key doesn't exist - if cmd.baseCmd.Err() == Nil { + // This happens when the key doesn't exist. + // Use rawErr() (not Err()): readReply runs inside the batch's Exec, before + // the autopipeline batch's done channel is closed, so Err()->await() would + // deadlock on the very Exec that is calling readReply. + if cmd.baseCmd.rawErr() == Nil { cmd.val = "" return Nil } // Handle other base command errors - if cmd.baseCmd.Err() != nil { - return cmd.baseCmd.Err() + if cmd.baseCmd.rawErr() != nil { + return cmd.baseCmd.rawErr() } if readType, err := rd.PeekReplyType(); err != nil { @@ -212,6 +219,7 @@ func NewJSONSliceCmd(ctx context.Context, args ...interface{}) *JSONSliceCmd { } func (cmd *JSONSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -220,15 +228,19 @@ func (cmd *JSONSliceCmd) SetVal(val []interface{}) { } func (cmd *JSONSliceCmd) Val() []interface{} { + cmd.await() return cmd.val } func (cmd *JSONSliceCmd) Result() ([]interface{}, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *JSONSliceCmd) readReply(rd *proto.Reader) error { - if cmd.baseCmd.Err() == Nil { + // rawErr(), not Err(): readReply runs inside Exec before the batch's done + // channel closes, so Err()->await() would deadlock (see JSONCmd.readReply). + if cmd.baseCmd.rawErr() == Nil { cmd.val = nil return Nil } @@ -238,7 +250,7 @@ func (cmd *JSONSliceCmd) readReply(rd *proto.Reader) error { } else if readType == proto.RespArray { response, err := rd.ReadReply() if err != nil { - return nil + return err } else { cmd.val = response.([]interface{}) } @@ -299,6 +311,7 @@ func NewIntPointerSliceCmd(ctx context.Context, args ...interface{}) *IntPointer } func (cmd *IntPointerSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -307,10 +320,12 @@ func (cmd *IntPointerSliceCmd) SetVal(val []*int64) { } func (cmd *IntPointerSliceCmd) Val() []*int64 { + cmd.await() return cmd.val } func (cmd *IntPointerSliceCmd) Result() ([]*int64, error) { + cmd.await() return cmd.val, cmd.err } diff --git a/backend/vendor/github.com/redis/go-redis/v9/list_commands.go b/backend/vendor/github.com/redis/go-redis/v9/list_commands.go index 9d9e16c6..afaf58e1 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/list_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/list_commands.go @@ -34,6 +34,8 @@ type ListCmdable interface { RPushX(ctx context.Context, key string, values ...interface{}) *IntCmd LMove(ctx context.Context, source, destination, srcpos, destpos string) *StringCmd BLMove(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration) *StringCmd + LMoveM(ctx context.Context, source, destination, srcpos, destpos string, args LMoveMArgs) *StringSliceCmd + BLMoveM(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration, args LMoveMArgs) *StringSliceCmd } func (c cmdable) BLPop(ctx context.Context, timeout time.Duration, keys ...string) *StringSliceCmd { @@ -156,6 +158,45 @@ type LPosArgs struct { Rank, MaxLen int64 } +// LMoveMMode is the count semantics for LMOVEM/BLMOVEM. +type LMoveMMode string + +const ( + LMoveMCount LMoveMMode = "COUNT" // up to Count + LMoveMExactly LMoveMMode = "EXACTLY" // exactly Count, or nothing +) + +// LMoveMOrder is the destination ordering for LMOVEM/BLMOVEM. +type LMoveMOrder string + +const ( + LMoveMOBO LMoveMOrder = "OBO" // one-by-one, order reversed + LMoveMBulk LMoveMOrder = "BULK" // preserve order +) + +// LMoveMArgs configures the optional count group of LMOVEM/BLMOVEM. +// Count <= 0 moves a single element. Mode defaults to COUNT, Order to BULK. +type LMoveMArgs struct { + Mode LMoveMMode + Count int64 + Order LMoveMOrder +} + +func (a LMoveMArgs) appendArgs(args []interface{}) []interface{} { + if a.Count <= 0 { + return args + } + mode := a.Mode + if mode == "" { + mode = LMoveMCount + } + order := a.Order + if order == "" { + order = LMoveMBulk + } + return append(args, string(mode), a.Count, string(order)) +} + func (c cmdable) LPos(ctx context.Context, key string, value string, a LPosArgs) *IntCmd { args := []interface{}{"lpos", key, value} if a.Rank != 0 { @@ -295,3 +336,27 @@ func (c cmdable) BLMove( _ = c(ctx, cmd) return cmd } + +// LMoveM atomically moves multiple elements between lists (Redis 8.10+). +// srcpos/destpos are "LEFT" or "RIGHT". Returns moved elements, or redis.Nil if none. +func (c cmdable) LMoveM(ctx context.Context, source, destination, srcpos, destpos string, a LMoveMArgs) *StringSliceCmd { + args := make([]interface{}, 5, 8) + args[0], args[1], args[2], args[3], args[4] = "lmovem", source, destination, srcpos, destpos + args = a.appendArgs(args) + cmd := NewStringSliceCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// BLMoveM is the blocking variant of LMoveM (Redis 8.10+); timeout 0 blocks forever. +// Returns moved elements, or redis.Nil on timeout. +func (c cmdable) BLMoveM(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration, a LMoveMArgs) *StringSliceCmd { + args := make([]interface{}, 6, 9) + args[0], args[1], args[2], args[3], args[4] = "blmovem", source, destination, srcpos, destpos + args[5] = formatSec(ctx, timeout) + args = a.appendArgs(args) + cmd := NewStringSliceCmd(ctx, args...) + cmd.setReadTimeout(timeout) + _ = c(ctx, cmd) + return cmd +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/local_cache.go b/backend/vendor/github.com/redis/go-redis/v9/local_cache.go new file mode 100644 index 00000000..2a5579c6 --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/local_cache.go @@ -0,0 +1,774 @@ +package redis + +import ( + "context" + "math" + "sync" + "sync/atomic" + "time" +) + +// cacheEntryState tracks the lifecycle of a local cache entry. +type cacheEntryState uint8 + +const ( + // cacheEntryInProgress marks a placeholder entry while a value is being fetched. + cacheEntryInProgress cacheEntryState = iota + // cacheEntryValid marks an entry that contains a value that can be returned. + cacheEntryValid +) + +// cacheEntry represents a cached command reply and its Redis-key associations. +type cacheEntry struct { + cacheKey string + redisKeys []string + value []byte + state cacheEntryState + + token uint64 + sizeBytes int64 + reservedAt time.Time + waitCh chan struct{} + waitClosed bool + + // lastAccessNs is a recency token for LRU eviction: a global atomic counter + // bumped on every access, stored atomically so the read path can mark a + // touch under the shard's RLock without upgrading to a write lock. + lastAccessNs atomic.Int64 + + // validAt retains time.Now's monotonic component for the MaxStaleness + // backstop, so wall-clock corrections cannot extend an entry's lifetime. + // Written under Lock (Set/Fulfill), read under RLock (get). + validAt time.Time + + // ownerConnID is the conn that fetched this entry (set by FulfillOwned; 0 = + // none). Default CLIENT TRACKING sends a key's invalidation only to that + // conn, so the entry must be evicted when it goes away (see EvictByConn). + ownerConnID uint64 +} + +// lruSequence is the global monotonic counter feeding lastAccessNs. It totally +// orders recency across all entries in all shards for approximate-LRU eviction. +var lruSequence atomic.Int64 + +// nextLRUToken returns the next strictly-greater LRU token. +func nextLRUToken() int64 { + return lruSequence.Add(1) +} + +// CacheSizer calculates estimated memory usage in bytes for a cache entry. +// +// Experimental: this API may change in a minor release. +type CacheSizer func(cacheKey string, redisKeys []string, value []byte) int64 + +// CacheConfig configures a local cache instance. +// +// Experimental: this API may change in a minor release. +type CacheConfig struct { + // MaxEntries limits the number of entries. Zero or negative means unlimited. + MaxEntries int + // MaxMemoryBytes limits estimated memory usage in bytes. Zero or negative means unlimited. + // + // If both MaxEntries and MaxMemoryBytes are unlimited, MaxEntries defaults to + // defaultCacheMaxEntries so the cache cannot grow without bound. The cache is + // sharded 16 ways (above small thresholds) and each shard enforces its 1/16 + // share, so an entry larger than MaxMemoryBytes/16 is never admitted — + // size it to at least 16× your largest reply. + MaxMemoryBytes int64 + // Sizer estimates memory usage per entry. If nil, a built-in approximation is used. + // + // Sizer may be invoked concurrently from multiple goroutines and must be + // thread-safe. It must return quickly and must not call back into the + // cache (Get, Set, Delete*, Flush, etc.): some call sites hold an internal + // shard lock, so re-entry can deadlock. + Sizer CacheSizer + // StaleTimeout is the duration after which an IN_PROGRESS placeholder is + // considered stale and eligible for takeover by a new Reserve call. + // If zero, defaults to defaultStaleTimeout (5s). + StaleTimeout time.Duration + + // DrainInterval is the background-drainer period (default 5ms; zero uses the + // default): how often idle pool conns are swept for buffered "invalidate" + // frames, roughly bounding cache-hit staleness. Values below 1ms are clamped + // to 1ms. + DrainInterval time.Duration + + // MaxStaleness caps how long a cached entry is served after it became valid, + // regardless of invalidation. It is a correctness + // BACKSTOP for lost invalidations or connection-lifecycle gaps ("Window 2"), not + // the primary freshness mechanism. Keep it well above the invalidation round-trip + // (e.g. seconds); per-entry refetch overhead scales ~1/MaxStaleness. + // + // Default: 0 (disabled). + MaxStaleness time.Duration +} + +// Cache is the thread-safe storage contract used by client-side caching. +// +// All methods may be called concurrently. Cache keys and Redis keys are opaque +// strings and must be preserved exactly. Removing a reservation must wake any +// Get calls waiting for it. +// +// Reserve must allow only one caller to fetch a missing key and return a token +// that is valid until FulfillOwned, Cancel, or an eviction removes that +// reservation. FulfillOwned and Cancel must modify only a reservation with the +// matching token. Get may wait for an in-progress reservation and must stop +// waiting when ctx is done. +// +// Experimental: this API may change in a minor release. +type Cache interface { + Get(ctx context.Context, cacheKey string) ([]byte, bool) + Reserve(cacheKey string, redisKeys []string) (token uint64, shouldFetch bool) + // FulfillOwned publishes a reserved value and records the connection that + // fetched it so the entry can be evicted if that connection loses tracking. + FulfillOwned(cacheKey string, token, ownerConnID uint64, value []byte) bool + Cancel(cacheKey string, token uint64) bool + DeleteByRedisKey(redisKey string) int + DeleteByCacheKey(cacheKey string) bool + // EvictByConn removes every entry fetched by connID. + EvictByConn(connID uint64) int + Flush() int +} + +const ( + defaultStaleTimeout = 5 * time.Second + defaultCacheShardCount = 16 + + // defaultCacheMaxEntries bounds the cache when the config leaves both + // MaxEntries and MaxMemoryBytes unlimited (matches the 10k-entry default + // other Redis clients use, e.g. redis-py). + defaultCacheMaxEntries = 10000 + + // shardingThresholdEntries / shardingThresholdBytes: caches with capacity + // below these thresholds fall back to a single shard so global LRU / + // memory-cap semantics behave exactly as a non-sharded cache would. + shardingThresholdEntries = 64 + shardingThresholdBytes = 64 * 1024 +) + +// NewLocalCache creates a thread-safe local cache with approximate-LRU +// eviction. The cache is internally sharded by cache-key hash to reduce +// mutex contention under high concurrent access. +// +// Experimental: this API may change in a minor release. +func NewLocalCache(cfg CacheConfig) *LocalCache { + sizer := cfg.Sizer + if sizer == nil { + sizer = defaultCacheSizer + } + + staleTimeout := cfg.StaleTimeout + if staleTimeout <= 0 { + staleTimeout = defaultStaleTimeout + } + + maxEntries := cfg.MaxEntries + maxMemoryBytes := cfg.MaxMemoryBytes + // An unbounded cache can grow until the process OOMs; require at least + // one limit. + if maxEntries <= 0 && maxMemoryBytes <= 0 { + maxEntries = defaultCacheMaxEntries + } + + shardCount := defaultCacheShardCount + if maxEntries > 0 && maxEntries < shardingThresholdEntries { + shardCount = 1 + } + if maxMemoryBytes > 0 && maxMemoryBytes < int64(shardingThresholdBytes) { + shardCount = 1 + } + + c := &LocalCache{ + shards: make([]cacheShard, shardCount), + shardCount: uint32(shardCount), + shardMask: uint32(shardCount - 1), + sizer: sizer, + } + for i := range c.shards { + s := &c.shards[i] + s.entries = make(map[string]*cacheEntry) + s.byRedisKey = make(map[string]map[string]struct{}) + s.byConnID = make(map[uint64]map[string]struct{}) + // Distribute capacity so the per-shard caps sum to exactly the + // configured limits; a ceil-per-shard split would let total residency + // exceed MaxEntries/MaxMemoryBytes. + if maxEntries > 0 { + s.maxEntries = maxEntries / shardCount + if i < maxEntries%shardCount { + s.maxEntries++ + } + } + if maxMemoryBytes > 0 { + s.maxMemoryBytes = maxMemoryBytes / int64(shardCount) + if int64(i) < maxMemoryBytes%int64(shardCount) { + s.maxMemoryBytes++ + } + } + s.maxStaleness = cfg.MaxStaleness + s.sizer = sizer + s.staleTimeout = staleTimeout + } + return c +} + +// LocalCache is the built-in sharded approximate-LRU cache. +// +// Experimental: this API may change in a minor release. +type LocalCache struct { + shards []cacheShard + shardCount uint32 + shardMask uint32 + sizer CacheSizer + + nextToken atomic.Uint64 + hits atomic.Uint64 + misses atomic.Uint64 +} + +var _ Cache = (*LocalCache)(nil) + +// cacheShard holds the state for one shard of LocalCache. The mutex +// protects entries, byRedisKey, byConnID, and usedBytes. +type cacheShard struct { + mu sync.RWMutex + entries map[string]*cacheEntry + byRedisKey map[string]map[string]struct{} + // byConnID is the owning-conn reverse index (twin of byRedisKey): conn id -> + // its cache keys. Populated by FulfillOwned, cleaned in removeEntryLocked, + // consumed by EvictByConn. + byConnID map[uint64]map[string]struct{} + usedBytes int64 + + maxEntries int + maxMemoryBytes int64 + maxStaleness time.Duration + sizer CacheSizer + staleTimeout time.Duration +} + +// shardFor returns the shard responsible for cacheKey. +func (c *LocalCache) shardFor(cacheKey string) *cacheShard { + if c.shardCount == 1 { + return &c.shards[0] + } + return &c.shards[fnv1a32(cacheKey)&c.shardMask] +} + +// fnv1a32 returns the FNV-1a 32-bit hash of s. Allocation-free. +func fnv1a32(s string) uint32 { + const ( + offset uint32 = 2166136261 + prime uint32 = 16777619 + ) + h := offset + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= prime + } + return h +} + +const defaultCacheEntryOverhead int64 = 96 + +func defaultCacheSizer(cacheKey string, redisKeys []string, value []byte) int64 { + size := defaultCacheEntryOverhead + int64(len(cacheKey)+len(value)) + for _, key := range redisKeys { + size += int64(len(key)) + 16 + } + if size < 0 { + return 0 + } + return size +} + +// Get returns a copy of a cached value, waiting for an in-progress fetch when +// necessary. +func (c *LocalCache) Get(ctx context.Context, cacheKey string) ([]byte, bool) { + if ctx == nil { + ctx = context.Background() + } + value, ok := c.shardFor(cacheKey).get(ctx, cacheKey) + if ok { + c.hits.Add(1) + } else { + c.misses.Add(1) + } + return value, ok +} + +// get is the read-side hot path. Holds only the shard's read lock; updates +// the LRU recency timestamp via atomic store on the entry — no write-lock +// upgrade is needed. +func (s *cacheShard) get(ctx context.Context, cacheKey string) ([]byte, bool) { + for { + s.mu.RLock() + entry, ok := s.entries[cacheKey] + if !ok { + s.mu.RUnlock() + return nil, false + } + + if entry.state == cacheEntryInProgress { + waitCh := entry.waitCh + // Bound the wait by the placeholder's remaining stale window so an + // abandoned reservation cannot block waiters indefinitely. + remaining := s.staleTimeout - time.Since(entry.reservedAt) + s.mu.RUnlock() + if waitCh == nil { + // Defensive: treat a missing waitCh as a miss to avoid busy-looping. + return nil, false + } + if remaining <= 0 { + // Placeholder already stale; miss so the caller refetches. + return nil, false + } + // Wait for the in-flight fetch to either publish (Fulfill) or abort (Cancel/Delete/Flush). + timer := time.NewTimer(remaining) + select { + case <-waitCh: + timer.Stop() + case <-ctx.Done(): + timer.Stop() + return nil, false + case <-timer.C: + return nil, false + } + continue + } + + if entry.state != cacheEntryValid { + s.mu.RUnlock() + return nil, false + } + + // Max-staleness backstop: a Valid entry older than maxStaleness is treated + // as a miss and evicted, so a lost invalidation or connection-lifecycle + // staleness (Window 2) cannot keep a stale value resident past MaxStaleness. + // Evict under the write lock so the next access re-fetches — a stale-but-present + // entry would otherwise suppress the re-fetch via Reserve. + if s.maxStaleness > 0 && time.Since(entry.validAt) > s.maxStaleness { + s.mu.RUnlock() + s.mu.Lock() + if cur, ok := s.entries[cacheKey]; ok && cur == entry { + s.removeEntryLocked(cacheKey) + } + s.mu.Unlock() + return nil, false + } + + value := cloneBytes(entry.value) + // Record access timestamp without upgrading the lock. Last writer + // wins; cross-goroutine ordering of timestamps is fine for + // approximate-LRU semantics. + entry.lastAccessNs.Store(nextLRUToken()) + s.mu.RUnlock() + return value, true + } +} + +// Stats returns cumulative activity and current residency. +func (c *LocalCache) Stats() CSCStats { + return CSCStats{ + Hits: c.hits.Load(), + Misses: c.misses.Load(), + Entries: c.Len(), + MemoryUsageBytes: c.MemoryUsage(), + } +} + +// Reserve claims a missing cache key for fetching. +func (c *LocalCache) Reserve(cacheKey string, redisKeys []string) (token uint64, shouldFetch bool) { + keysCopy := cloneStrings(redisKeys) + waitCh := make(chan struct{}) + reservedAt := time.Now() + sizeBytes := c.sizer(cacheKey, keysCopy, nil) + if sizeBytes < 0 { + sizeBytes = 0 + } + newToken := c.nextToken.Add(1) + + s := c.shardFor(cacheKey) + s.mu.Lock() + defer s.mu.Unlock() + + if entry, ok := s.entries[cacheKey]; ok { + switch entry.state { + case cacheEntryValid: + // Existing-VALID hit: record access; caller will re-Get to + // retrieve. + entry.lastAccessNs.Store(nextLRUToken()) + return 0, false + case cacheEntryInProgress: + if time.Since(entry.reservedAt) < s.staleTimeout { + return 0, false + } + s.removeEntryLocked(cacheKey) + default: + return 0, false + } + } + + if s.maxMemoryBytes > 0 && sizeBytes > s.maxMemoryBytes { + return 0, true + } + + entry := &cacheEntry{ + cacheKey: cacheKey, + redisKeys: keysCopy, + state: cacheEntryInProgress, + token: newToken, + reservedAt: reservedAt, + waitCh: waitCh, + sizeBytes: sizeBytes, + } + entry.lastAccessNs.Store(nextLRUToken()) + + s.setEntryLocked(entry) + // Evict only Valid victims. If still over capacity the shard holds only + // in-flight placeholders: rather than abort a peer's fetch, drop this + // reservation (the caller fetches uncached). The hard cap holds either way. + s.evictValidLocked() + if s.overCapacityLocked() { + s.removeEntryLocked(cacheKey) + return 0, true + } + if s.entries[cacheKey] != entry { + return 0, true + } + return newToken, true +} + +// FulfillOwned publishes a reserved value and records ownerConnID so +// EvictByConn can drop it when that connection is removed. ownerConnID == 0 +// leaves the value unowned. +func (c *LocalCache) FulfillOwned(cacheKey string, token, ownerConnID uint64, value []byte) bool { + return c.fulfill(cacheKey, token, ownerConnID, value) +} + +func (c *LocalCache) fulfill(cacheKey string, token, ownerConnID uint64, value []byte) bool { + valueCopy := cloneBytes(value) + + s := c.shardFor(cacheKey) + s.mu.Lock() + defer s.mu.Unlock() + + entry, ok := s.entries[cacheKey] + if !ok || entry.state != cacheEntryInProgress || entry.token != token { + return false + } + + valueSize := s.sizer(cacheKey, entry.redisKeys, valueCopy) + if valueSize < 0 { + valueSize = 0 + } + if s.maxMemoryBytes > 0 && valueSize > s.maxMemoryBytes { + s.removeEntryLocked(cacheKey) + return false + } + + s.usedBytes += valueSize - entry.sizeBytes + entry.value = valueCopy + entry.sizeBytes = valueSize + entry.state = cacheEntryValid + entry.validAt = time.Now() + entry.token = 0 + entry.lastAccessNs.Store(nextLRUToken()) + if ownerConnID != 0 { + entry.ownerConnID = ownerConnID + s.indexConnLocked(ownerConnID, cacheKey) + } + s.closeWaitersLocked(entry) + + s.evictIfNeededLocked() + current, stillExists := s.entries[cacheKey] + return stillExists && current == entry && entry.state == cacheEntryValid +} + +// EvictByConn removes every entry fetched by connID and returns the count. +// Called when a conn is removed/swapped: the server stops delivering those +// keys' invalidations, so keeping them risks stale serves. Errs toward a miss. +func (c *LocalCache) EvictByConn(connID uint64) int { + if connID == 0 { + return 0 + } + removed := 0 + for i := range c.shards { + removed += c.shards[i].evictByConn(connID) + } + return removed +} + +func (s *cacheShard) evictByConn(connID uint64) int { + s.mu.Lock() + defer s.mu.Unlock() + + cacheKeys, ok := s.byConnID[connID] + if !ok { + return 0 + } + toRemove := make([]string, 0, len(cacheKeys)) + for cacheKey := range cacheKeys { + toRemove = append(toRemove, cacheKey) + } + removed := 0 + for _, cacheKey := range toRemove { + if s.removeEntryLocked(cacheKey) { + removed++ + } + } + return removed +} + +// indexConnLocked records cacheKey under connID in the owning-connection index. +func (s *cacheShard) indexConnLocked(connID uint64, cacheKey string) { + cacheKeys := s.byConnID[connID] + if cacheKeys == nil { + cacheKeys = make(map[string]struct{}) + s.byConnID[connID] = cacheKeys + } + cacheKeys[cacheKey] = struct{}{} +} + +// Cancel removes the reservation matching token. +func (c *LocalCache) Cancel(cacheKey string, token uint64) bool { + s := c.shardFor(cacheKey) + s.mu.Lock() + defer s.mu.Unlock() + + entry, ok := s.entries[cacheKey] + if !ok || entry.state != cacheEntryInProgress || entry.token != token { + return false + } + + s.removeEntryLocked(cacheKey) + return true +} + +// DeleteByRedisKey removes entries associated with redisKey. +func (c *LocalCache) DeleteByRedisKey(redisKey string) int { + removed := 0 + for i := range c.shards { + removed += c.shards[i].deleteByRedisKey(redisKey) + } + return removed +} + +func (s *cacheShard) deleteByRedisKey(redisKey string) int { + s.mu.Lock() + defer s.mu.Unlock() + + cacheKeys, ok := s.byRedisKey[redisKey] + if !ok { + return 0 + } + + // Remove IN_PROGRESS placeholders too: an invalidation can arrive on a + // different stream than the in-flight reply (the background drainer), so the + // fetch may predate the write. Removing makes the racing Fulfill fail and + // waiters refetch, so a raced-invalidation value is never published. + toRemove := make([]string, 0, len(cacheKeys)) + for cacheKey := range cacheKeys { + toRemove = append(toRemove, cacheKey) + } + + removed := 0 + for _, cacheKey := range toRemove { + if s.removeEntryLocked(cacheKey) { + removed++ + } + } + return removed +} + +// DeleteByCacheKey removes one entry by its internal cache key. +func (c *LocalCache) DeleteByCacheKey(cacheKey string) bool { + s := c.shardFor(cacheKey) + s.mu.Lock() + defer s.mu.Unlock() + return s.removeEntryLocked(cacheKey) +} + +// Flush removes all entries. +func (c *LocalCache) Flush() int { + removed := 0 + for i := range c.shards { + removed += c.shards[i].flush() + } + return removed +} + +func (s *cacheShard) flush() int { + s.mu.Lock() + defer s.mu.Unlock() + + // Flush placeholders too (see deleteByRedisKey): a flush (FLUSHDB, or the + // owned-cache flush on Close) means everything, including in-flight fetches, + // may be stale. + removed := 0 + for cacheKey := range s.entries { + if s.removeEntryLocked(cacheKey) { + removed++ + } + } + return removed +} + +// Len returns the current number of entries and reservations. +func (c *LocalCache) Len() int { + n := 0 + for i := range c.shards { + s := &c.shards[i] + s.mu.RLock() + n += len(s.entries) + s.mu.RUnlock() + } + return n +} + +// MemoryUsage returns the cache's estimated memory usage in bytes. +func (c *LocalCache) MemoryUsage() int64 { + var total int64 + for i := range c.shards { + s := &c.shards[i] + s.mu.RLock() + total += s.usedBytes + s.mu.RUnlock() + } + return total +} + +func (s *cacheShard) setEntryLocked(entry *cacheEntry) { + if old, exists := s.entries[entry.cacheKey]; exists { + s.removeEntryLocked(old.cacheKey) + } + + s.entries[entry.cacheKey] = entry + s.usedBytes += entry.sizeBytes + + for _, redisKey := range entry.redisKeys { + cacheKeys := s.byRedisKey[redisKey] + if cacheKeys == nil { + cacheKeys = make(map[string]struct{}) + s.byRedisKey[redisKey] = cacheKeys + } + cacheKeys[entry.cacheKey] = struct{}{} + } +} + +func (s *cacheShard) removeEntryLocked(cacheKey string) bool { + entry, exists := s.entries[cacheKey] + if !exists { + return false + } + + delete(s.entries, cacheKey) + s.usedBytes -= entry.sizeBytes + if s.usedBytes < 0 { + s.usedBytes = 0 + } + + for _, redisKey := range entry.redisKeys { + cacheKeys := s.byRedisKey[redisKey] + if cacheKeys == nil { + continue + } + delete(cacheKeys, cacheKey) + if len(cacheKeys) == 0 { + delete(s.byRedisKey, redisKey) + } + } + + if entry.ownerConnID != 0 { + if cacheKeys := s.byConnID[entry.ownerConnID]; cacheKeys != nil { + delete(cacheKeys, cacheKey) + if len(cacheKeys) == 0 { + delete(s.byConnID, entry.ownerConnID) + } + } + } + + s.closeWaitersLocked(entry) + return true +} + +func (s *cacheShard) closeWaitersLocked(entry *cacheEntry) { + if entry.waitCh != nil && !entry.waitClosed { + close(entry.waitCh) + entry.waitClosed = true + } +} + +func (s *cacheShard) overCapacityLocked() bool { + if s.maxEntries > 0 && len(s.entries) > s.maxEntries { + return true + } + if s.maxMemoryBytes > 0 && s.usedBytes > s.maxMemoryBytes { + return true + } + return false +} + +// evictIfNeededLocked evicts by approximate LRU (O(N) scan; rare in +// well-sized caches) until under capacity. Used by Set/Fulfill: it prefers a +// Valid victim but falls back to the oldest IN_PROGRESS placeholder to keep the +// hard cap (that placeholder's Fulfill then fails and its waiters refetch). +func (s *cacheShard) evictIfNeededLocked() { + for s.overCapacityLocked() { + victim := s.oldestLocked(cacheEntryValid) + if victim == nil { + victim = s.oldestLocked(cacheEntryInProgress) + } + if victim == nil { + return + } + s.removeEntryLocked(victim.cacheKey) + } +} + +// evictValidLocked evicts only Valid entries until under capacity. Unlike +// evictIfNeededLocked it never evicts a placeholder, so Reserve can't abort a +// peer's in-flight fetch. +func (s *cacheShard) evictValidLocked() { + for s.overCapacityLocked() { + victim := s.oldestLocked(cacheEntryValid) + if victim == nil { + return + } + s.removeEntryLocked(victim.cacheKey) + } +} + +// oldestLocked returns the entry in the given state with the smallest +// lastAccessNs (the least-recently-used), or nil when none exists. +func (s *cacheShard) oldestLocked(state cacheEntryState) *cacheEntry { + var victim *cacheEntry + var oldestNs int64 = math.MaxInt64 + for _, e := range s.entries { + if e.state != state { + continue + } + if ns := e.lastAccessNs.Load(); ns < oldestNs { + oldestNs = ns + victim = e + } + } + return victim +} + +func cloneBytes(src []byte) []byte { + if src == nil { + return nil + } + dst := make([]byte, len(src)) + copy(dst, src) + return dst +} + +func cloneStrings(src []string) []string { + if len(src) == 0 { + return nil + } + dst := make([]string, len(src)) + copy(dst, src) + return dst +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/manager.go b/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/manager.go index ff54c717..bf32389a 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/manager.go +++ b/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/manager.go @@ -77,17 +77,33 @@ type Manager struct { activeOperationCount atomic.Int64 // Number of active operations closed atomic.Bool // Manager closed state + // shutdownTimeout bounds each pool hook's Shutdown during Close. A field + // (not a constant) so tests can exercise the failed-Close-then-retry + // path without waiting out the real budget. + shutdownTimeout time.Duration + // Notification hooks for extensibility hooks []NotificationHook hooksMu sync.RWMutex // Protects hooks slice poolHooksRef *PoolHook + // additionalPoolHooks are pool hooks bound to pools other than the primary + // one (e.g. a dedicated pipeline connection pool). Each is an independent + // *PoolHook bound to its own pool because the hook's failed-handoff removal + // target (HandoffRequest.Pool) is taken from the hook's single pool field, + // so one hook cannot safely serve two pools. They share this Manager as + // their operations manager, keeping MOVING/MIGRATING tracking centralized. + additionalPoolHooks []additionalPoolHook + // Connections that successfully enabled maintnotifications. These need to be // retired before the pool-level listeners are removed. maintNotificationsConns sync.Map // connID -> *pool.Conn - // Cluster state reload callback for SMIGRATED notifications - clusterStateReloadCallback ClusterStateReloadCallback + // Cluster state reload callback for SMIGRATED notifications. + // Stored atomically because it is set from the OnNewNode hook while a node + // client is being created and read from the SMIGRATED push handler on that + // node's connections, which can overlap during connection init. + clusterStateReloadCallback atomic.Pointer[ClusterStateReloadCallback] } // MovingOperation tracks an active MOVING operation. @@ -113,11 +129,12 @@ func NewManager(client interfaces.ClientInterface, pool pool.Pooler, config *Con } hm := &Manager{ - client: client, - pool: pool, - options: client.GetOptions(), - config: config.Clone(), - hooks: make([]NotificationHook, 0), + client: client, + pool: pool, + options: client.GetOptions(), + config: config.Clone(), + hooks: make([]NotificationHook, 0), + shutdownTimeout: 10 * time.Second, } // Set up push notification handling @@ -134,6 +151,78 @@ func (hm *Manager) InitPoolHook(baseDialer func(context.Context, string, string) hm.pool.AddPoolHook(poolHook) } +// additionalPoolHook pairs a pool hook with the pool it was attached to so the +// manager can shut it down and detach it on Close. +type additionalPoolHook struct { + pool pool.Pooler + hook *PoolHook +} + +// InitPoolHookForPool attaches a maintnotifications pool hook to an additional +// pool (e.g. a client's dedicated pipeline connection pool). A fresh, independent +// *PoolHook is created and bound to the given pool so that connections which fail +// handoff are removed from the correct pool — the hook's removal target is its own +// single pool field, so the primary hook cannot be reused for a second pool. The +// new hook shares this Manager as its operations manager, so MOVING/MIGRATING +// tracking and notification handling stay centralized across both pools. +func (hm *Manager) InitPoolHookForPool(p pool.Pooler, baseDialer func(context.Context, string, string) (net.Conn, error)) { + if p == nil { + return + } + poolSize := 0 + network := "" + if hm.options != nil { + poolSize = hm.options.GetPoolSize() + network = hm.options.GetNetwork() + } + hook := NewPoolHookWithPoolSize(baseDialer, network, hm.config, hm, poolSize) + hook.SetPool(p) + // The closed check, the append, AND the AddPoolHook must all be atomic with + // respect to Close's snapshot: hold the lock across all three so either Close + // runs first (closed==true here, so we neither register nor attach) or we + // register+attach first (Close's snapshot then includes this hook and tears + // it down). Attaching outside the lock left a window where Close could + // snapshot/tear-down between the append and the attach, then AddPoolHook + // would re-attach to a closed manager's pool — leaking an active hook. + // p.AddPoolHook is lock-free (atomic swap on the pool's own hook manager) and + // never calls back into this manager, so holding hooksMu across it is safe. + hm.hooksMu.Lock() + defer hm.hooksMu.Unlock() + if hm.closed.Load() { + return + } + hm.additionalPoolHooks = append(hm.additionalPoolHooks, additionalPoolHook{pool: p, hook: hook}) + p.AddPoolHook(hook) +} + +// hookForConn returns the pool hook that owns cn's pool: an additional hook +// when the connection came from a secondary pool (e.g. a client's dedicated +// pipeline pool), the primary hook otherwise. Handoffs must be queued through +// the owning hook — the HandoffRequest carries that hook's pool, and a failed +// handoff removes the connection from it, so queuing a pipeline-pool +// connection on the primary hook would close the connection without freeing +// its slot in the pipeline pool's bookkeeping. +func (hm *Manager) hookForConn(cn *pool.Conn) *PoolHook { + if cn == nil { + return hm.poolHooksRef + } + name := cn.PoolName() + if name == "" { + return hm.poolHooksRef + } + hm.hooksMu.RLock() + defer hm.hooksMu.RUnlock() + for _, ah := range hm.additionalPoolHooks { + // Pooler does not expose the name; the concrete pool does. A Pooler + // implementation without it simply never matches and falls through to + // the primary hook — the pre-existing behavior. + if np, ok := ah.pool.(interface{ Name() string }); ok && np.Name() == name { + return ah.hook + } + } + return hm.poolHooksRef +} + // setupPushNotifications sets up push notification handling by registering with the client's processor. func (hm *Manager) setupPushNotifications() error { processor := hm.client.GetPushProcessor() @@ -284,17 +373,38 @@ func (hm *Manager) maintNotificationsConnSnapshot() []*pool.Conn { func (hm *Manager) retireMaintNotificationsConns(ctx context.Context) { conns := hm.maintNotificationsConnSnapshot() - if len(conns) == 0 || hm.pool == nil { + if len(conns) == 0 { return } - if retirer, ok := hm.pool.(pool.ConnRetirer); ok { - retirer.RetireConns(ctx, conns, pool.CloseReasonMaintNotificationsDisabled) - return + // Tracked connections can live in the primary pool OR in any additional + // pool this manager attached a hook to (e.g. a client's dedicated pipeline + // connection pool — its conns run initConn and are tracked exactly like + // primary ones). Retire through every pool: RetireConns skips connections + // a pool does not own, so offering the full snapshot to each pool is safe. + // Missing the additional pools left pipeline connections in service with + // maintnotifications enabled but no hook attached after a runtime + // downgrade — pushes on them were silently dropped. + pools := make([]pool.Pooler, 0, 1+len(hm.additionalPoolHooks)) + if hm.pool != nil { + pools = append(pools, hm.pool) } + hm.hooksMu.RLock() + for _, ah := range hm.additionalPoolHooks { + if ah.pool != nil { + pools = append(pools, ah.pool) + } + } + hm.hooksMu.RUnlock() - for _, cn := range conns { - _ = hm.pool.CloseConn(ctx, cn, pool.CloseReasonMaintNotificationsDisabled, pool.MetricStateIdle) + for _, pl := range pools { + if retirer, ok := pl.(pool.ConnRetirer); ok { + retirer.RetireConns(ctx, conns, pool.CloseReasonMaintNotificationsDisabled) + continue + } + for _, cn := range conns { + _ = pl.CloseConn(ctx, cn, pool.CloseReasonMaintNotificationsDisabled, pool.MetricStateIdle) + } } } @@ -312,7 +422,7 @@ func (hm *Manager) Close() error { // Shutdown the pool hook if it exists if hm.poolHooksRef != nil { // Use a timeout to prevent hanging indefinitely - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + shutdownCtx, cancel := context.WithTimeout(context.Background(), hm.shutdownTimeout) defer cancel() err := hm.poolHooksRef.Shutdown(shutdownCtx) @@ -327,6 +437,36 @@ func (hm *Manager) Close() error { } } + // Shutdown and detach any hooks bound to additional pools (e.g. a dedicated + // pipeline pool). Snapshot under the lock so we don't iterate concurrently + // with a registering InitPoolHookForPool; Shutdown itself runs unlocked. + hm.hooksMu.Lock() + additional := hm.additionalPoolHooks + hm.additionalPoolHooks = nil + hm.hooksMu.Unlock() + for i, ah := range additional { + shutdownCtx, cancel := context.WithTimeout(context.Background(), hm.shutdownTimeout) + err := ah.hook.Shutdown(shutdownCtx) + cancel() + if err != nil { + // Could not cleanly shut down this hook. Put it and the ones not + // yet processed back so a retried Close still sees them, then stay + // open so the caller can retry, matching the primary-hook behavior + // above. Hooks before i already shut down and detached. + hm.hooksMu.Lock() + remaining := make([]additionalPoolHook, 0, len(additional)-i+len(hm.additionalPoolHooks)) + remaining = append(remaining, additional[i:]...) + remaining = append(remaining, hm.additionalPoolHooks...) + hm.additionalPoolHooks = remaining + hm.hooksMu.Unlock() + hm.closed.Store(false) + return err + } + if ah.pool != nil { + ah.pool.RemovePoolHook(ah.hook) + } + } + // Clear all active operations hm.activeMovingOps.Range(func(key, value interface{}) bool { hm.activeMovingOps.Delete(key) @@ -401,13 +541,13 @@ func (hm *Manager) AddNotificationHook(notificationHook NotificationHook) { // SetClusterStateReloadCallback sets the callback function that will be called when a SMIGRATED notification is received. // This allows node clients to notify their parent ClusterClient to reload cluster state. func (hm *Manager) SetClusterStateReloadCallback(callback ClusterStateReloadCallback) { - hm.clusterStateReloadCallback = callback + hm.clusterStateReloadCallback.Store(&callback) } // TriggerClusterStateReload calls the cluster state reload callback if it's set. // This is called when a SMIGRATED notification is received. func (hm *Manager) TriggerClusterStateReload(ctx context.Context, hostPort string, slotRanges []string) { - if hm.clusterStateReloadCallback != nil { - hm.clusterStateReloadCallback(ctx, hostPort, slotRanges) + if cb := hm.clusterStateReloadCallback.Load(); cb != nil { + (*cb)(ctx, hostPort, slotRanges) } } diff --git a/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/pool_hook.go b/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/pool_hook.go index 752abc71..71d14a7f 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/pool_hook.go +++ b/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/pool_hook.go @@ -164,12 +164,17 @@ func (ph *PoolHook) OnPut(ctx context.Context, conn *pool.Conn) (shouldPool bool } if err := conn.MarkQueuedForHandoff(); err != nil { - // If marking fails, check if handoff was processed in the meantime + // Marking can fail if a worker advanced the connection's state between + // our queueHandoff above and here. Re-check ShouldHandoff: with the CAS + // rollback in Conn.MarkQueuedForHandoff, a worker that already cleared + // the handoff state is no longer misreported as ShouldHandoff=true, so a + // cleared connection is reliably detected and pooled here. if !conn.ShouldHandoff() { - // Handoff was processed - this is normal, pool the connection + // Handoff was processed - this is normal, pool the connection. return true, false, nil } - // Other error - remove the connection + // Still marked for handoff in an ambiguous state — remove it rather than + // returning a connection a queued worker may still close or replace. return false, true, nil } internal.Logger.Printf(ctx, logs.MarkedForHandoff(conn.GetID())) diff --git a/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/push_notification_handler.go b/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/push_notification_handler.go index 7108265b..26ae7fd4 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/push_notification_handler.go +++ b/backend/vendor/github.com/redis/go-redis/v9/maintnotifications/push_notification_handler.go @@ -161,8 +161,15 @@ func (snh *NotificationHandler) handleMoving(ctx context.Context, handlerCtx pus // If the connection is in use (StateInUse), it will be queued when returned to the pool via OnPut. // This handles the case where the connection is idle and might never be retrieved again. if poolConn.GetStateMachine().GetState() == pool.StateIdle { - if snh.manager.poolHooksRef != nil && snh.manager.poolHooksRef.workerManager != nil { - if err := snh.manager.poolHooksRef.workerManager.queueHandoff(poolConn); err != nil { + // Queue on the hook that owns this connection's pool, not + // unconditionally on the primary one: the request carries the + // hook's pool, and a failed handoff removes the connection + // from it — for a dedicated pipeline-pool connection the + // primary pool cannot do that, which would close the + // connection while leaving a dead slot behind. + owner := snh.manager.hookForConn(poolConn) + if owner != nil && owner.workerManager != nil { + if err := owner.workerManager.queueHandoff(poolConn); err != nil { internal.Logger.Printf(context.Background(), logs.FailedToQueueHandoff(poolConn.GetID(), err)) } else { // Mark the connection as queued for handoff to prevent it from being retrieved diff --git a/backend/vendor/github.com/redis/go-redis/v9/options.go b/backend/vendor/github.com/redis/go-redis/v9/options.go index ba45a0cb..0863b386 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/options.go +++ b/backend/vendor/github.com/redis/go-redis/v9/options.go @@ -16,6 +16,7 @@ import ( "time" "github.com/redis/go-redis/v9/auth" + "github.com/redis/go-redis/v9/internal" "github.com/redis/go-redis/v9/internal/pool" "github.com/redis/go-redis/v9/internal/proto" "github.com/redis/go-redis/v9/internal/util" @@ -121,12 +122,12 @@ type Options struct { // MinRetryBackoff is the minimum backoff between each retry. // -1 disables backoff. // - // default: 8 milliseconds + // default: 10 milliseconds MinRetryBackoff time.Duration // MaxRetryBackoff is the maximum backoff between each retry. // -1 disables backoff. - // default: 512 milliseconds; + // default: 1 second; MaxRetryBackoff time.Duration // DialTimeout for establishing new connections. @@ -157,7 +158,7 @@ type Options struct { // - `-1` - no timeout (block indefinitely). // - `-2` - disables SetReadDeadline calls completely. // - // default: 3 seconds + // default: 5 seconds ReadTimeout time.Duration // WriteTimeout for socket writes. If reached, commands will fail @@ -166,7 +167,7 @@ type Options struct { // - `-1` - no timeout (block indefinitely). // - `-2` - disables SetWriteDeadline calls completely. // - // default: 3 seconds + // default: 5 seconds (same as ReadTimeout, which it follows when unset) WriteTimeout time.Duration // ContextTimeoutEnabled controls whether the client respects context timeouts and deadlines. @@ -187,6 +188,86 @@ type Options struct { // default: 32KiB (32768 bytes) WriteBufferSize int + // PipelineReadBufferSize is the size of the bufio.Reader buffer for pipeline connections. + // If set to a value > 0, a separate connection pool will be created specifically for + // pipelining operations (Pipeline, AutoPipeline and AsyncAutoPipeline) with + // this buffer size. + // + // This allows you to use large buffers for pipelining (to reduce syscalls and improve + // throughput) while keeping regular command buffers small (to save memory). + // + // If not set (0), pipeline operations will use the regular connection pool with + // ReadBufferSize buffers. + // + // Recommended: 64–128 KiB for high-throughput pipelining. The benefit here is + // on the READ side: a batch's replies arrive as one large stream, and a bigger + // buffer consumes them in fewer syscalls instead of refilling repeatedly + // mid-batch. Size it to roughly the reply volume of a typical batch — which + // for read-heavy pipelines is dominated by value sizes, not command count. + // (The write-side counterpart, sizing to the outgoing wire bytes so the batch + // flushes without overflowing mid-write, belongs to PipelineWriteBufferSize.) + // Benchmarks show throughput climbs from the 32 KiB default up to ~64 KiB and + // then plateaus; going beyond ~128 KiB gives no further gain and very large + // buffers (≥512 KiB) can regress throughput and waste memory. Bigger is not + // better. + // + // Example: + // client := redis.NewClient(&redis.Options{ + // Addr: "localhost:6379", + // ReadBufferSize: 32 * 1024, // 32 KiB for regular commands + // PipelineReadBufferSize: 128 * 1024, // 128 KiB for pipelining + // PipelineWriteBufferSize: 128 * 1024, + // }) + // + // Memory impact: With PoolSize=100 and PipelinePoolSize=10: + // - Without pipeline pool: 100 conns × 128 KiB = 12.8 MB (if all use 128 KiB buffers) + // - With pipeline pool: (100 × 32 KiB) + (10 × 128 KiB) = 4.5 MB (~65% savings) + // + // default: 0 (use ReadBufferSize) + PipelineReadBufferSize int + + // PipelineWriteBufferSize is the size of the bufio.Writer buffer for pipeline connections. + // If set to a value > 0, a separate connection pool will be created specifically for + // pipelining operations (Pipeline, AutoPipeline and AsyncAutoPipeline) with + // this buffer size. + // + // This allows you to use large buffers for pipelining (to reduce syscalls and improve + // throughput) while keeping regular command buffers small (to save memory). + // + // If not set (0), pipeline operations will use the regular connection pool with + // WriteBufferSize buffers. + // + // Recommended: 64–128 KiB for high-throughput pipelining (size to roughly + // MaxBatchSize × average-command-bytes). Throughput plateaus past ~64 KiB and + // gains nothing beyond ~128 KiB; very large buffers (≥512 KiB) can regress it. + // See PipelineReadBufferSize for the full rationale. + // + // default: 0 (use WriteBufferSize) + PipelineWriteBufferSize int + + // PipelinePoolSize is the pool size for the separate pipeline connection pool. + // Only used if PipelineReadBufferSize or PipelineWriteBufferSize is set. + // + // Pipelining typically needs fewer connections than regular operations because + // batching reduces connection contention. A smaller pool saves memory while + // maintaining high throughput. + // + // If not set (0), defaults to 10 connections. + // + // default: 10 + PipelinePoolSize int + + // AutoPipelineOptions is the default config for BOTH autopipeliner faces: + // AutoPipeline and AsyncAutoPipeline use it when called without an + // explicit config, falling back to their per-face defaults + // (DefaultBlockingAutoPipelineOptions / DefaultAutoPipelineOptions) when it + // is nil. Pass a config to either method to override. Commands issued + // through an autopipeliner are batched into pipelines to cut round-trips + // and raise throughput. + // + // EXPERIMENTAL: this API is subject to change, use with caution. + AutoPipelineOptions *AutoPipelineOptions + // PoolFIFO type of connection pool. // // - true for FIFO pool @@ -300,6 +381,8 @@ type Options struct { // PushNotificationProcessor is the processor for handling push notifications. // If nil, a default processor will be created for RESP3 connections. + // With client-side caching, a custom processor runs while an idle connection + // is borrowed from the pool and should return promptly. PushNotificationProcessor push.NotificationProcessor // FailingTimeoutSeconds is the timeout in seconds for marking a cluster node as failing. @@ -313,12 +396,76 @@ type Options struct { // transitions seamlessly. Requires Protocol: 3 (RESP3) for push notifications. // If nil, maintnotifications are in "auto" mode and will be enabled if the server supports it. MaintNotificationsConfig *maintnotifications.Config + + // ClientSideCacheConfig enables client-side caching when non-nil. Together + // with ClientSideCache it is the on/off switch for the feature: leave both + // nil to disable CSC, set either one to enable it. If ClientSideCache is also set, it + // takes precedence over this config. + // + // Client-side caching is disabled when CredentialsProvider, + // CredentialsProviderContext, or StreamingCredentialsProvider is set: + // provider-backed credentials can change the ACL identity after the cache + // namespace is selected. Fixed Username/Password values are supported and + // included in the cache namespace. + // + // Experimental: this API may change in a minor release. + ClientSideCacheConfig *ClientSideCacheConfig + + // ClientSideCache is an explicit Cache implementation used for client-side + // caching. When set, it overrides ClientSideCacheConfig. Intended for + // advanced users that want to share a cache across clients or supply a + // custom implementation. + // + // A shared Cache is only safe across clients on the same server and DB. + // Clients with different fixed Username/Password values are isolated by a + // username namespace. + // Client-side caching is restricted to DB 0 and disabled with a warning + // otherwise. It is also disabled with any credential provider; see + // ClientSideCacheConfig. + // + // Experimental: this API may change in a minor release. + ClientSideCache Cache + + // ClientSideCacheStrategy selects the invalidation architecture used when + // client-side caching is enabled (via ClientSideCacheConfig or + // ClientSideCache); it is ignored when CSC is disabled. The zero value is + // CSCStrategySharedTracking, currently the only implemented strategy. + // + // Experimental: this API may change in a minor release. + ClientSideCacheStrategy CSCStrategy } +// CSCStrategy selects the client-side caching invalidation architecture. Set via +// Options.ClientSideCacheStrategy; fixed for the client's lifetime. +// +// CSCStrategySharedTracking is currently the only implemented strategy; the type +// exists as an extension point for additional architectures (e.g. a BCAST sidecar) +// without a breaking API change. +// +// Experimental: this API may change in a minor release. +type CSCStrategy int + +const ( + // CSCStrategySharedTracking (default, the zero value): one shared cache; every + // pool connection runs plain CLIENT TRACKING ON and a background drainer applies + // buffered invalidations. Portable (no BCAST), and matches the other Redis clients. + CSCStrategySharedTracking CSCStrategy = iota +) + func (opt *Options) init() { if opt.Addr == "" { opt.Addr = "localhost:6379" } + // An unknown strategy would thread the CSC gates inconsistently (e.g. tracking + // on with no drainer), serving stale data. Clamp to the only supported value. + switch opt.ClientSideCacheStrategy { + case CSCStrategySharedTracking: + default: + internal.Logger.Printf(context.Background(), + "redis: unknown ClientSideCacheStrategy %d; falling back to CSCStrategySharedTracking", + opt.ClientSideCacheStrategy) + opt.ClientSideCacheStrategy = CSCStrategySharedTracking + } if opt.Network == "" { if strings.HasPrefix(opt.Addr, "/") { opt.Network = "unix" @@ -357,6 +504,13 @@ func (opt *Options) init() { } if opt.ReadBufferSize == 0 { opt.ReadBufferSize = proto.DefaultBufferSize + } else if opt.Protocol == 3 && opt.ReadBufferSize < proto.MinRESP3ReadBufferSize { + // Too small to hold a push header, the processor would consume frames before + // knowing their name and could swallow a Pub/Sub frame. Clamp to the minimum. + internal.Logger.Printf(context.Background(), + "redis: ReadBufferSize=%d is below the RESP3 minimum %d; clamping.", + opt.ReadBufferSize, proto.MinRESP3ReadBufferSize) + opt.ReadBufferSize = proto.MinRESP3ReadBufferSize } if opt.WriteBufferSize == 0 { opt.WriteBufferSize = proto.DefaultBufferSize @@ -367,7 +521,7 @@ func (opt *Options) init() { case -1: opt.ReadTimeout = 0 case 0: - opt.ReadTimeout = 3 * time.Second + opt.ReadTimeout = 5 * time.Second } switch opt.WriteTimeout { case -2: @@ -400,19 +554,24 @@ func (opt *Options) init() { case -1: opt.MinRetryBackoff = 0 case 0: - opt.MinRetryBackoff = 8 * time.Millisecond + opt.MinRetryBackoff = 10 * time.Millisecond } switch opt.MaxRetryBackoff { case -1: opt.MaxRetryBackoff = 0 case 0: - opt.MaxRetryBackoff = 512 * time.Millisecond + opt.MaxRetryBackoff = time.Second } if opt.FailingTimeoutSeconds == 0 { opt.FailingTimeoutSeconds = 15 } + if opt.Protocol == 2 && (opt.ClientSideCache != nil || opt.ClientSideCacheConfig != nil) { + internal.Logger.Printf(context.Background(), + "redis: client-side caching requires Protocol: 3 (RESP3); caching is disabled") + } + opt.MaintNotificationsConfig = opt.MaintNotificationsConfig.ApplyDefaultsWithPoolConfig(opt.PoolSize, opt.MaxActiveConns) // auto-detect endpoint type if not specified @@ -442,13 +601,24 @@ func (opt *Options) NewDialer() func(context.Context, string, string) (net.Conn, return NewDialer(opt) } +// defaultKeepAliveConfig is the TCP keep-alive policy of the default dialers +// here and in sentinel.go: start probing after 30s idle (below typical LB/NAT +// idle timeouts), then declare the peer dead after 3 unanswered probes 5s +// apart. +var defaultKeepAliveConfig = net.KeepAliveConfig{ + Enable: true, + Idle: 30 * time.Second, + Interval: 5 * time.Second, + Count: 3, +} + // NewDialer returns a function that will be used as the default dialer // when none is specified in Options.Dialer. func NewDialer(opt *Options) func(context.Context, string, string) (net.Conn, error) { return func(ctx context.Context, network, addr string) (net.Conn, error) { netDialer := &net.Dialer{ - Timeout: opt.DialTimeout, - KeepAlive: 5 * time.Minute, + Timeout: opt.DialTimeout, + KeepAliveConfig: defaultKeepAliveConfig, } if opt.TLSConfig == nil { return netDialer.DialContext(ctx, network, addr) @@ -547,10 +717,12 @@ func setupTCPConn(u *url.URL) (*Options, error) { // a host and a port. If the host is missing, it defaults to localhost // and if the port is missing, it defaults to 6379. func getHostPortWithDefaults(u *url.URL) (string, string) { - host, port, err := net.SplitHostPort(u.Host) - if err != nil { - host = u.Host - } + // u.Hostname and u.Port strip the surrounding brackets from IPv6 literals + // (e.g. "[::1]" -> "::1") and handle the missing-port case, which + // net.SplitHostPort instead reports as an error. Relying on them avoids + // leaving the brackets on the host, which the caller's net.JoinHostPort + // would wrap again and turn "redis://[::1]" into "[[::1]]:6379". + host, port := u.Hostname(), u.Port() if host == "" { host = "localhost" } @@ -627,6 +799,10 @@ func (o *queryOptions) duration(name string) time.Duration { } dur, err := time.ParseDuration(s) if err == nil { + if dur <= 0 { + // disable timeouts + return -1 + } return dur } if o.err == nil { diff --git a/backend/vendor/github.com/redis/go-redis/v9/osscluster.go b/backend/vendor/github.com/redis/go-redis/v9/osscluster.go index efd52960..c6f97a40 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/osscluster.go +++ b/backend/vendor/github.com/redis/go-redis/v9/osscluster.go @@ -142,6 +142,19 @@ type ClusterOptions struct { // default: 32KiB (32768 bytes) WriteBufferSize int + // PipelineReadBufferSize, PipelineWriteBufferSize and PipelinePoolSize + // configure an optional separate connection pool used for pipelining on + // each node, with its own (typically larger) buffers. See the same-named + // fields on Options for details. The pool is created only when PipelineReadBufferSize or PipelineWriteBufferSize is set (PipelinePoolSize alone does not enable it). + PipelineReadBufferSize int + PipelineWriteBufferSize int + PipelinePoolSize int + + // AutoPipelineOptions is the default config for BOTH autopipeliner faces + // (AutoPipeline and AsyncAutoPipeline), applied when they are called + // without explicit options. See Options.AutoPipelineOptions. + AutoPipelineOptions *AutoPipelineOptions + TLSConfig *tls.Config // DisableRoutingPolicies disables the request/response policy routing system. @@ -190,7 +203,9 @@ type ClusterOptions struct { ShardPicker routing.ShardPicker // ClusterStateReloadInterval is the interval for reloading the cluster state. - // Default is 10 seconds. + // MOVED/ASK redirects still trigger an immediate reactive reload, so this + // only bounds how stale a topology can get without traffic errors. + // Default is 60 seconds. ClusterStateReloadInterval time.Duration } @@ -235,7 +250,7 @@ func (opt *ClusterOptions) init() { case -1: opt.ReadTimeout = 0 case 0: - opt.ReadTimeout = 3 * time.Second + opt.ReadTimeout = 5 * time.Second } switch opt.WriteTimeout { case -1: @@ -251,13 +266,13 @@ func (opt *ClusterOptions) init() { case -1: opt.MinRetryBackoff = 0 case 0: - opt.MinRetryBackoff = 8 * time.Millisecond + opt.MinRetryBackoff = 10 * time.Millisecond } switch opt.MaxRetryBackoff { case -1: opt.MaxRetryBackoff = 0 case 0: - opt.MaxRetryBackoff = 512 * time.Millisecond + opt.MaxRetryBackoff = time.Second } if opt.NewClient == nil { @@ -273,7 +288,7 @@ func (opt *ClusterOptions) init() { } if opt.ClusterStateReloadInterval == 0 { - opt.ClusterStateReloadInterval = 10 * time.Second + opt.ClusterStateReloadInterval = 60 * time.Second } } @@ -455,11 +470,15 @@ func (opt *ClusterOptions) clientOptions() *Options { ConnMaxLifetimeJitter: opt.ConnMaxLifetimeJitter, ReadBufferSize: opt.ReadBufferSize, WriteBufferSize: opt.WriteBufferSize, - DisableIdentity: opt.DisableIdentity, - DisableIndentity: opt.DisableIdentity, - IdentitySuffix: opt.IdentitySuffix, - FailingTimeoutSeconds: opt.FailingTimeoutSeconds, - TLSConfig: opt.TLSConfig, + + PipelineReadBufferSize: opt.PipelineReadBufferSize, + PipelineWriteBufferSize: opt.PipelineWriteBufferSize, + PipelinePoolSize: opt.PipelinePoolSize, + DisableIdentity: opt.DisableIdentity, + DisableIndentity: opt.DisableIndentity, + IdentitySuffix: opt.IdentitySuffix, + FailingTimeoutSeconds: opt.FailingTimeoutSeconds, + TLSConfig: opt.TLSConfig, // If ClusterSlots is populated, then we probably have an artificial // cluster whose nodes are not in clustering mode (otherwise there isn't // much use for ClusterSlots config). This means we cannot execute the @@ -477,13 +496,13 @@ func (opt *ClusterOptions) clientOptions() *Options { type clusterNode struct { Client *Client - latency uint32 // atomic - generation uint32 // atomic - failing uint32 // atomic - loaded uint32 // atomic + latency atomic.Uint32 + generation atomic.Uint32 + failing atomic.Uint32 + loaded atomic.Uint32 // last time the latency measurement was performed for the node, stored in nanoseconds from epoch - lastLatencyMeasurement int64 // atomic + lastLatencyMeasurement atomic.Int64 } func newClusterNodeWithNodeAddress(clOpt *ClusterOptions, addr, nodeAddress string) *clusterNode { @@ -494,7 +513,7 @@ func newClusterNodeWithNodeAddress(clOpt *ClusterOptions, addr, nodeAddress stri Client: clOpt.NewClient(opt), } - node.latency = math.MaxUint32 + node.latency.Store(math.MaxUint32) if clOpt.RouteByLatency { go node.updateLatency() } @@ -536,46 +555,46 @@ func (n *clusterNode) updateLatency() { } else { latency = float64(dur) / float64(successes) } - atomic.StoreUint32(&n.latency, uint32(latency+0.5)) + n.latency.Store(uint32(latency + 0.5)) n.SetLastLatencyMeasurement(time.Now()) } func (n *clusterNode) Latency() time.Duration { - latency := atomic.LoadUint32(&n.latency) + latency := n.latency.Load() return time.Duration(latency) * time.Microsecond } func (n *clusterNode) MarkAsFailing() { - atomic.StoreUint32(&n.failing, uint32(time.Now().Unix())) - atomic.StoreUint32(&n.loaded, 0) + n.failing.Store(uint32(time.Now().Unix())) + n.loaded.Store(0) } func (n *clusterNode) Failing() bool { timeout := int64(n.Client.opt.FailingTimeoutSeconds) - failing := atomic.LoadUint32(&n.failing) + failing := n.failing.Load() if failing == 0 { return false } if time.Now().Unix()-int64(failing) < timeout { return true } - atomic.StoreUint32(&n.failing, 0) + n.failing.Store(0) return false } func (n *clusterNode) Generation() uint32 { - return atomic.LoadUint32(&n.generation) + return n.generation.Load() } func (n *clusterNode) LastLatencyMeasurement() int64 { - return atomic.LoadInt64(&n.lastLatencyMeasurement) + return n.lastLatencyMeasurement.Load() } func (n *clusterNode) SetGeneration(gen uint32) { for { - v := atomic.LoadUint32(&n.generation) - if gen < v || atomic.CompareAndSwapUint32(&n.generation, v, gen) { + v := n.generation.Load() + if gen < v || n.generation.CompareAndSwap(v, gen) { break } } @@ -583,15 +602,15 @@ func (n *clusterNode) SetGeneration(gen uint32) { func (n *clusterNode) SetLastLatencyMeasurement(t time.Time) { for { - v := atomic.LoadInt64(&n.lastLatencyMeasurement) - if t.UnixNano() < v || atomic.CompareAndSwapInt64(&n.lastLatencyMeasurement, v, t.UnixNano()) { + v := n.lastLatencyMeasurement.Load() + if t.UnixNano() < v || n.lastLatencyMeasurement.CompareAndSwap(v, t.UnixNano()) { break } } } func (n *clusterNode) Loading() bool { - loaded := atomic.LoadUint32(&n.loaded) + loaded := n.loaded.Load() if loaded == 1 { return false } @@ -603,7 +622,7 @@ func (n *clusterNode) Loading() bool { err := n.Client.Ping(ctx).Err() loading := err != nil && isLoadingError(err) if !loading { - atomic.StoreUint32(&n.loaded, 1) + n.loaded.Store(1) } return loading } @@ -620,7 +639,7 @@ type clusterNodes struct { closed bool onNewNode []func(rdb *Client) - generation uint32 // atomic + generation atomic.Uint32 } func newClusterNodes(opt *ClusterOptions) *clusterNodes { @@ -685,7 +704,7 @@ func (c *clusterNodes) Addrs() ([]string, error) { } func (c *clusterNodes) NextGeneration() uint32 { - return atomic.AddUint32(&c.generation, 1) + return c.generation.Add(1) } // GC removes unused nodes. @@ -1064,8 +1083,8 @@ type clusterStateHolder struct { reloadInterval time.Duration state atomic.Value - reloading uint32 // atomic - reloadPending uint32 // atomic - set to 1 when reload is requested during active reload + reloading atomic.Uint32 + reloadPending atomic.Uint32 // set to 1 when reload is requested during active reload } func newClusterStateHolder(load func(ctx context.Context) (*clusterState, error), reloadInterval time.Duration) *clusterStateHolder { @@ -1086,8 +1105,8 @@ func (c *clusterStateHolder) Reload(ctx context.Context) (*clusterState, error) func (c *clusterStateHolder) LazyReload() { // If already reloading, mark that another reload is pending - if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { - atomic.StoreUint32(&c.reloadPending, 1) + if !c.reloading.CompareAndSwap(0, 1) { + c.reloadPending.Store(1) return } @@ -1095,22 +1114,22 @@ func (c *clusterStateHolder) LazyReload() { for { _, err := c.Reload(context.Background()) if err != nil { - atomic.StoreUint32(&c.reloadPending, 0) - atomic.StoreUint32(&c.reloading, 0) + c.reloadPending.Store(0) + c.reloading.Store(0) return } // Clear pending flag after reload completes, before cooldown // This captures notifications that arrived during the reload - atomic.StoreUint32(&c.reloadPending, 0) + c.reloadPending.Store(0) // Wait cooldown period time.Sleep(200 * time.Millisecond) // Check if another reload was requested during cooldown - if atomic.LoadUint32(&c.reloadPending) == 0 { + if c.reloadPending.Load() == 0 { // No pending reload, we're done - atomic.StoreUint32(&c.reloading, 0) + c.reloading.Store(0) return } @@ -1153,6 +1172,17 @@ type ClusterClient struct { cmdInfoResolver *commandInfoResolver cmdable hooksMixin + + // himport is the cluster-wide HIMPORT fieldset registry, shared with + // every node client (masters and replicas alike — roles change with the + // topology) so any connection serving an HIMPORT SET can lazily replay + // the PREPARE (see himport.go, himport_cluster.go). + himport *himportRegistry + + autopipelinerMu *sync.Mutex // guards the autopipeliner fields against concurrent first-call creation + autopipeliner *AutoPipeliner // blocking face (ClusterClient.AutoPipeline) + asyncAutopipeliner *AutoPipeliner // deferred face (ClusterClient.AsyncAutoPipeline) + autopipelinerClosed bool // set by Close: refuse to resurrect a pipeliner on a closed client } // NewClusterClient returns a Redis Cluster client as described in @@ -1165,10 +1195,19 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { opt.init() c := &ClusterClient{ - opt: opt, - nodes: newClusterNodes(opt), + opt: opt, + nodes: newClusterNodes(opt), + himport: newHImportRegistry(), + autopipelinerMu: &sync.Mutex{}, } + // Every node client shares the cluster-wide fieldset registry, replicas + // included: a promoted replica's connections carry no prepared flags, so + // the first HIMPORT SET routed to it replays the PREPARE lazily. + c.nodes.OnNewNode(func(nodeClient *Client) { + nodeClient.himport = c.himport + }) + c.cmdsInfoCache = newCmdsInfoCache(c.cmdsInfo) c.state = newClusterStateHolder(c.loadState, opt.ClusterStateReloadInterval) @@ -1223,7 +1262,26 @@ func (c *ClusterClient) ReloadState(ctx context.Context) { // It is rare to Close a ClusterClient, as the ClusterClient is meant // to be long-lived and shared between many goroutines. func (c *ClusterClient) Close() error { - return c.nodes.Close() + // Stop both cached autopipeliners (blocking and async faces) before + // closing nodes, so its background flusher goroutines don't outlive the + // client. AutoPipeliner.Close is idempotent and nil-safe here. + c.autopipelinerMu.Lock() + ap, async := c.autopipeliner, c.asyncAutopipeliner + c.autopipeliner, c.asyncAutopipeliner = nil, nil + c.autopipelinerClosed = true // getters refuse to resurrect on a closed client + c.autopipelinerMu.Unlock() + var firstErr error + for _, p := range []*AutoPipeliner{ap, async} { + if p != nil { + if err := p.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + } + if err := c.nodes.Close(); err != nil && firstErr == nil { + firstErr = err + } + return firstErr } func (c *ClusterClient) Process(ctx context.Context, cmd Cmder) error { @@ -1555,6 +1613,155 @@ func (c *ClusterClient) Pipeline() Pipeliner { return &pipe } +// clusterAutoPipelineOptions applies the cluster shard-count default: commands +// are routed to shards by slot (see installAutoPipelineSharding), so unlike a +// standalone client — which defaults to a single deep queue — a cluster client +// wants several shards to keep concurrent nodes' batches separate. The caller's +// config is copied before the default is filled in, never mutated. +func clusterAutoPipelineOptions(cfg *AutoPipelineOptions) *AutoPipelineOptions { + c2 := *cfg + if c2.NumShards == 0 { + c2.NumShards = numAutoPipelineShards() + } + // A cluster always routes by slot, so per-key order holds regardless of shard + // count; mark it so construction's NumShards ordering check (which targets + // round-robin sharding) does not reject the cluster default or an explicit + // NumShards on the deferred (async) face. + c2.contentSharded = true + return &c2 +} + +// AutoPipeline returns the blocking autopipeliner for this cluster client: each +// command call blocks until executed (drop-in shape) while the engine batches +// concurrent callers into pipelines. Commands keep per-goroutine order; across +// nodes, ordering is per key (slot routing keeps a key on one shard and node +// sub-pipelines execute concurrently). Use AutoPipelineWithOptions to override +// DefaultBlockingAutoPipelineOptions. Cached/shared; first call's config wins. +// Close it (or the client) to release its goroutines. +// +// It returns an error if the supplied config is invalid (e.g. MaxConcurrentBatches>1 +// without Unordered, or a negative size); on error no instance is cached. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *ClusterClient) AutoPipeline() (*AutoPipeliner, error) { + return c.AutoPipelineWithOptions(nil) +} + +// AutoPipelineWithOptions is AutoPipeline with explicit options instead of +// ClusterOptions.AutoPipelineOptions / the default. Cached/shared; first call wins. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *ClusterClient) AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return getOrCreateAutoPipeliner(c.autopipelinerMu, &c.autopipeliner, &c.autopipelinerClosed, nil, config, + func() *AutoPipelineOptions { + if c.opt.AutoPipelineOptions != nil { + return c.opt.AutoPipelineOptions + } + return DefaultBlockingAutoPipelineOptions() + }, + func(cfg *AutoPipelineOptions) (*AutoPipeliner, error) { + ap, err := newAutoPipeliner(c, clusterAutoPipelineOptions(cfg), true) + if err != nil { + return nil, err + } + c.installAutoPipelineSharding(ap) + return ap, nil + }) +} + +// installAutoPipelineSharding routes commands to shards by cluster slot so each +// shard's batch lands on a single master node, keeping per-node pipelines deep +// instead of splitting every batch across all nodes at flush. Cluster slots are +// contiguous per node, so bucketing by slot range (slot*shards/16384) keeps a +// node's slots together. Keyless commands hash to slot -1 → bucket 0; multi-node +// commands are already rejected from pipelines, so only single-node commands +// reach here. +func (c *ClusterClient) installAutoPipelineSharding(ap *AutoPipeliner) { + // Reject commands whose request policy cannot ride a pipeline (ReqAllNodes/ + // ReqAllShards/ReqMultiShard) at submit, BEFORE they can join a merged + // batch: mapCmdsByNode fails a whole mapping on such a command (user + // pipelines are all-or-nothing), and one autopipeline caller must not be + // able to poison unrelated callers' batches. Rejecting here also keeps the + // lone-command fast path consistent with batched dispatch — the command is + // refused regardless of what it happens to coalesce with. + ap.setPreflight(func(ctx context.Context, cmd Cmder) error { + if c.cmdInfoResolver == nil { + return nil + } + if policy := c.cmdInfoResolver.GetCommandPolicy(ctx, cmd); policy != nil && !policy.CanBeUsedInPipeline() { + return fmt.Errorf( + "redis: cannot pipeline command %q with request policy ReqAllNodes/ReqAllShards/ReqMultiShard; Note: This behavior is subject to change in the future", cmd.Name(), + ) + } + return nil + }) + // Commands whose routing is not slot-derived must not be coalesced: a solo + // flush reaches ClusterClient.process and its special handling (FT.CURSOR + // READ/DEL are sticky to the node holding the cursor), but inside a batch + // mapCmdsByNode routes by slot and can hit the wrong shard — visible only + // under concurrent traffic, which is the worst way to find it. Divert them + // instead of rejecting: they work fine on their own connection (review + // finding by codex on #3942). + ap.setMustDivert(func(ctx context.Context, cmd Cmder) bool { + if c.cmdInfoResolver == nil { + return false + } + policy := c.cmdInfoResolver.GetCommandPolicy(ctx, cmd) + return policy != nil && policy.Request == routing.ReqSpecial + }) + + const slots = 16384 + n := ap.numShards() + ap.setShardFn(func(cmd Cmder) int { + // Compute the exact slot once and cache it on the command; the flush + // router (mapCmdsByNode) reuses the cached value, so the slot is resolved + // once per command, not twice. Keyless (slot -1) buckets to shard 0. + slot := c.cmdSlot(cmd, -1) + if slot < 0 { + return 0 + } + return slot * n / slots + }) +} + +// AsyncAutoPipeline returns the deferred autopipeliner: command calls return +// immediately and the result accessors block. Submit a window then read results +// for the highest throughput. By default, +// ClusterOptions.AutoPipelineOptions is used if set, otherwise +// DefaultAutoPipelineOptions. Ordering across nodes is per key: slot routing +// keeps a key on one shard, and node sub-pipelines execute concurrently. Use +// AsyncAutoPipelineWithOptions to override. Cached/shared; first call's config wins. +// +// It returns an error if the supplied config is invalid (e.g. MaxConcurrentBatches>1 +// without Unordered, or a negative size); on error no instance is cached. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *ClusterClient) AsyncAutoPipeline() (*AutoPipeliner, error) { + return c.AsyncAutoPipelineWithOptions(nil) +} + +// AsyncAutoPipelineWithOptions is AsyncAutoPipeline with an explicit config +// instead of ClusterOptions.AutoPipelineOptions / the default. Cached/shared. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *ClusterClient) AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return getOrCreateAutoPipeliner(c.autopipelinerMu, &c.asyncAutopipeliner, &c.autopipelinerClosed, nil, config, + func() *AutoPipelineOptions { + if c.opt.AutoPipelineOptions != nil { + return c.opt.AutoPipelineOptions + } + return DefaultAutoPipelineOptions() + }, + func(cfg *AutoPipelineOptions) (*AutoPipeliner, error) { + ap, err := newAutoPipeliner(c, clusterAutoPipelineOptions(cfg), false) + if err != nil { + return nil, err + } + c.installAutoPipelineSharding(ap) + return ap, nil + }) +} + func (c *ClusterClient) Pipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().Pipelined(ctx, fn) } @@ -1638,9 +1845,18 @@ func (c *ClusterClient) mapCmdsByNode(ctx context.Context, cmdsMap *cmdsMap, cmd policy = c.cmdInfoResolver.GetCommandPolicy(ctx, cmd) } if policy != nil && !policy.CanBeUsedInPipeline() { - return fmt.Errorf( + // All-or-nothing: a user Pipeline() relies on the whole batch + // either dispatching or failing before anything executes, so a + // non-pipelineable command fails the entire mapping pre-dispatch. + // Autopipeline batches never reach here with such a command: the + // cluster face rejects them at submit (see the preflight installed + // by installAutoPipelineSharding), so one caller's bad command + // cannot poison a merged batch. + err := fmt.Errorf( "redis: cannot pipeline command %q with request policy ReqAllNodes/ReqAllShards/ReqMultiShard; Note: This behavior is subject to change in the future", cmd.Name(), ) + setCmdsErr(cmds, err) + return err } slot := c.cmdSlot(cmd, -1) var node *clusterNode @@ -1649,10 +1865,15 @@ func (c *ClusterClient) mapCmdsByNode(ctx context.Context, cmdsMap *cmdsMap, cmd if len(state.Masters) == 0 { return errClusterNoNodes } - // For read-only keyless commands, pick from all nodes (masters + slaves) - allNodes := append(state.Masters, state.Slaves...) - idx := c.opt.ShardPicker.Next(len(allNodes)) - node = allNodes[idx] + // For read-only keyless commands, pick from all nodes (masters + slaves). + // Index directly instead of building a combined slice, which would + // append into the shared snapshot's spare capacity and race. + idx := c.opt.ShardPicker.Next(len(state.Masters) + len(state.Slaves)) + if idx < len(state.Masters) { + node = state.Masters[idx] + } else { + node = state.Slaves[idx-len(state.Masters)] + } } else { node, err = c.slotReadOnlyNode(state, slot) if err != nil { @@ -1670,9 +1891,18 @@ func (c *ClusterClient) mapCmdsByNode(ctx context.Context, cmdsMap *cmdsMap, cmd policy = c.cmdInfoResolver.GetCommandPolicy(ctx, cmd) } if policy != nil && !policy.CanBeUsedInPipeline() { - return fmt.Errorf( + // All-or-nothing: a user Pipeline() relies on the whole batch + // either dispatching or failing before anything executes, so a + // non-pipelineable command fails the entire mapping pre-dispatch. + // Autopipeline batches never reach here with such a command: the + // cluster face rejects them at submit (see the preflight installed + // by installAutoPipelineSharding), so one caller's bad command + // cannot poison a merged batch. + err := fmt.Errorf( "redis: cannot pipeline command %q with request policy ReqAllNodes/ReqAllShards/ReqMultiShard; Note: This behavior is subject to change in the future", cmd.Name(), ) + setCmdsErr(cmds, err) + return err } slot := c.cmdSlot(cmd, -1) var node *clusterNode @@ -1707,31 +1937,76 @@ func (c *ClusterClient) cmdsAreReadOnly(ctx context.Context, cmds []Cmder) bool func (c *ClusterClient) processPipelineNode( ctx context.Context, node *clusterNode, cmds []Cmder, failedCmds *cmdsMap, ) { - _ = node.Client.withProcessPipelineHook(ctx, cmds, func(ctx context.Context, cmds []Cmder) error { - cn, err := node.Client.getConn(ctx) - if err != nil { + // This call runs on a per-node fan-out goroutine, so register it as an + // executor of every deferred-face batch among cmds: a NODE-level hook + // (OnNewNode — redisotel's tracing) reading a result before next() must + // get the not-yet-executed view from the accessor guards instead of + // blocking on a batch only this call chain completes (reproduced as a + // permanent wedge with a rediscmd-shaped Err() peek). + unregister := registerBatchExecutors(cmds) + defer unregister() + + // executed guards against a node-level hook short-circuiting (returning + // without calling next): the inner callback then never runs, and without + // surfacing the chain's error the cluster pipeline would report success + // for commands that were never sent. + executed := false + err := node.Client.withProcessPipelineHook(ctx, cmds, func(ctx context.Context, cmds []Cmder) error { + executed = true + // Acquire through the node's dedicated pipeline pool when one is + // configured (Pipeline*BufferSize propagate to node clients via + // clientOptions); withPipelineConn falls back to the main pool + // otherwise, preserving the previous behavior. entered distinguishes + // an acquisition failure (fn never ran) from an execution error. + entered := false + err := node.Client.withPipelineConn(ctx, func(ctx context.Context, cn *pool.Conn) error { + entered = true + return c.processPipelineNodeConn(ctx, node, cn, cmds, failedCmds) + }) + if err != nil && !entered { if !isContextError(err) { node.MarkAsFailing() } _ = c.mapCmdsByNode(ctx, failedCmds, cmds) setCmdsErr(cmds, err) - return err } - - var processErr error - defer func() { - node.Client.releaseConn(ctx, cn, processErr) - }() - processErr = c.processPipelineNodeConn(ctx, node, cn, cmds, failedCmds) - - return processErr + return err }) + if !executed { + // A hook returned without calling next. If it supplied an error that is + // a deliberate abort: set it and do not remap for retry (a retry would + // re-run the same hook). If it returned nil it short-circuited + // SUCCESSFULLY, having served the batch itself — the same thing a plain + // Pipeline hook may do — so setCmdsErr(nil) leaves the values it set + // intact (review finding by codex on #3942). + setCmdsErr(cmds, err) + return + } + if err != nil && cmdsFirstErr(cmds) == nil { + // Post-next verdict from a node-level hook on an all-clean sub-batch: + // the exec fully succeeded, so the error can only be the hook's own — + // apply it, mirroring AutoPipeliner.dispatchCmds. On a mixed batch the + // exec-recorded outcomes win (hooks conventionally echo next's error, + // and stamping the echo would overwrite successful replies). No remap: + // retrying would re-run the same hook. + setCmdsErr(cmds, err) + } } func (c *ClusterClient) processPipelineNodeConn( ctx context.Context, node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds *cmdsMap, ) error { + // HIMPORT bookkeeping: pending discards for this session and PREPAREs + // for registered fieldsets the batch references get written ahead of + // the batch (see himport.go). + injected := node.Client.himportInjectedCmds(ctx, cn, cmds) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } return writeCmds(wr, cmds) }); err != nil { if isBadConn(err, false, node.Client.getAddr()) { @@ -1745,18 +2020,56 @@ func (c *ClusterClient) processPipelineNodeConn( } return cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { - return c.pipelineReadCmds(ctx, node, rd, cmds, failedCmds) + if err := node.Client.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + // Transport error with the batch replies unread: same handling + // as a write error — the batch may be retried on a fresh + // connection. + if isBadConn(err, false, node.Client.getAddr()) { + node.MarkAsFailing() + } + if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + _ = c.mapCmdsByNode(ctx, failedCmds, cmds) + } + setCmdsErr(cmds, err) + return err + } + err := c.pipelineReadCmds(ctx, node, cn, rd, cmds, failedCmds) + if err == nil || isRedisError(err) { + node.Client.himportAfterBatch(cn, injected, cmds) + // SETs of registered fieldsets that lost their session state + // re-queue for the next attempt, which re-prepares lazily — + // the cluster equivalent of himportRetryFailedSets, bounded by + // the pipeline's attempt budget. A non-nil redis error here + // means pipelineReadCmds already re-queued the whole batch + // (retryable first-command error); adding the SETs again would + // duplicate them in the next attempt. + if err == nil { + c.himportRequeueFailedSets(ctx, cmds, failedCmds) + } + } + return err }) } func (c *ClusterClient) pipelineReadCmds( ctx context.Context, node *clusterNode, + cn *pool.Conn, rd *proto.Reader, cmds []Cmder, failedCmds *cmdsMap, ) error { for i, cmd := range cmds { + // Drain any buffered RESP3 push notifications before reading each + // reply — otherwise a push frame (e.g. a maintnotifications MOVING + // notification) is consumed AS the command's reply and every + // subsequent reply in the pipeline shifts by one command. The + // standalone pipeline and the cluster TxPipeline read loops already + // do this; this loop was the only push-blind reader, and the + // autopipeliner routes all cluster traffic through it. + if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) + } err := cmd.readReply(rd) cmd.SetErr(err) @@ -1781,7 +2094,8 @@ func (c *ClusterClient) pipelineReadCmds( } } - if err := cmds[0].Err(); err != nil && shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + // rawErr: execution path; never await an async command's batch here. + if err := cmds[0].rawErr(); err != nil && shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { _ = c.mapCmdsByNode(ctx, failedCmds, cmds) return err } @@ -1832,18 +2146,108 @@ func (c *ClusterClient) TxPipelined(ctx context.Context, fn func(Pipeliner) erro return c.TxPipeline().Pipelined(ctx, fn) } -func (c *ClusterClient) processTxPipeline(ctx context.Context, cmds []Cmder) error { - // Only call time.Now() if pipeline operation duration callback is set to avoid overhead +// A cluster tx pipeline sends MULTI, c1..cN, EXEC — N+2 commands, or N+3 with a +// leading ASKING — and always receives exactly that many replies, so every +// redirect/abort path leaves the connection clean. +// +// Possible reply sequences: +// 1. Slot owned here, no migration: +// +OK, +QUEUED x N, *N (array of N results) -> success +// 2. Slot already migrated away: +// +OK, -MOVED x N, -EXECABORT -> re-route whole tx +// 3. Slot in migrating state (still owned here, keys draining out). Per +// cmd, the queue reply is +QUEUED / -ASK / -TRYAGAIN (keys present / +// all gone / some gone); any -ASK or -TRYAGAIN dirties the tx, so +// EXEC is -EXECABORT. Still N+2 replies, like the cases above: +// +OK, (+QUEUED|-ASK|-TRYAGAIN) x N, -EXECABORT -> follow first redirect +// 4. Narrow race (all +QUEUED, slot moves before EXEC): +// +OK, +QUEUED x N, -MOVED -> re-route whole tx +// 5. Non-cluster command error (arity / ACL / unknown): +// +OK, +QUEUED..., -ERR..., -EXECABORT -> surface, not retryable +// 6. Narrow race (all +QUEUED, slot still migrating, keys drain before EXEC): +// +OK, +QUEUED x N, -ASK / -TRYAGAIN -> re-route on -ASK, back off on -TRYAGAIN +// +// EXEC reply — the reply that decides the outcome: +// +// *N success; read N per-command results +// -EXECABORT a queue-stage command failed; follow the first queue +// redirect (MOVED/ASK/TRYAGAIN), else surface the trigger +// -MOVED case 4; re-route whole tx to addr, reload topology +// -ASK race: slot entered migrating state; re-route to addr +// with a top-level ASKING before MULTI +// -TRYAGAIN race: migrating with split keys, or slot being trimmed +// (CLUSTER_REDIR_TRIMMING on a write); back off and retry +// the whole tx (same node still owns it) +// -CLUSTERDOWN cluster degraded; back off and retry whole tx +// +// ASK retry: the ASKING flag is NOT cleared between commands inside a MULTI +// so one top-level ASKING before MULTI covers the whole tx and lets the importing +// slot serve at EXEC. ASKING placed inside the MULTI would be queued and leave +// the flag unset during queueing, so the keyed commands would still get MOVED. +// +// Out of scope: WATCH's null-array EXEC and -CROSSSLOT; +// cluster TxPipeline is not used with WATCH and cross-slot is rejected client-side. + +type txOutcomeKind int + +const ( + txSuccess txOutcomeKind = iota // transaction executed; per-command results are set + txRetryMoved // MOVED: reload topology and re-route the whole tx + txRetryAsk // ASK: re-route to the target with a top-level ASKING + txRetryTryAgain // TRYAGAIN: back off and re-route the whole tx + txRetryConn // connection/write/read failure: re-route the whole tx + txFatal // non-retryable error; surface to the caller +) + +// txOutcome is the result of a single tx attempt. err is the error to report +// when the redirect/retry loop is exhausted (or the fatal error to surface); +// addr is the ASK target; execErr is the EXEC reply error used to mark +// aborted commands; unreadReplies forces the connection to be discarded +// when the read loop exited before consuming all N+2 replies, leaving bytes +// on the wire. +type txOutcome struct { + kind txOutcomeKind + err error + addr string + execErr error + unreadReplies bool +} + +// txRedirect records the first queue-stage redirect (MOVED/ASK/TRYAGAIN) seen +// while reading +QUEUED replies. Redis dirties and aborts the transaction on +// any such reply, so the EXEC reply will be EXECABORT and the client must +// follow the recorded redirect with the whole transaction. +type txRedirect struct { + moved bool + ask bool + tryAgain bool + addr string + err error +} + +// errTxDirtyConn forces releaseConn to discard a connection that may still have +// unread transaction replies on it (an early exit before consuming all N+2). +var errTxDirtyConn = errors.New("redis: connection has unread transaction replies") + +func (c *ClusterClient) processTxPipeline(ctx context.Context, cmds []Cmder) (retErr error) { var operationStart time.Time pipelineOpDurationCallback := otel.GetPipelineOperationDurationCallback() if pipelineOpDurationCallback != nil { operationStart = time.Now() } totalAttempts := 0 + var lastErr error + + defer func() { + if pipelineOpDurationCallback == nil { + return + } + finalErr := cmp.Or(retErr, cmdsFirstErr(cmds), lastErr) + pipelineOpDurationCallback(ctx, time.Since(operationStart), "MULTI", len(cmds), totalAttempts, finalErr, nil, 0) + }() // Trim multi .. exec. cmds = cmds[1 : len(cmds)-1] - if len(cmds) == 0 { return nil } @@ -1851,10 +2255,6 @@ func (c *ClusterClient) processTxPipeline(ctx context.Context, cmds []Cmder) err state, err := c.state.Get(ctx) if err != nil { setCmdsErr(cmds, err) - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), 1, err, nil, 0) - } return err } @@ -1866,77 +2266,85 @@ func (c *ClusterClient) processTxPipeline(ctx context.Context, cmds []Cmder) err case 1: for sl := range keyedCmdsBySlot { slot = sl - break } default: // TxPipeline does not support cross slot transaction. setCmdsErr(cmds, ErrCrossSlot) - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), 1, ErrCrossSlot, nil, 0) - } return ErrCrossSlot } node, err := state.slotMasterNode(slot) if err != nil { setCmdsErr(cmds, err) - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), 1, err, nil, 0) - } return err } - var lastErr error - cmdsMap := map[*clusterNode][]Cmder{node: cmds} + asking := false + // MOVED/ASK are routing changes, not transient failures: follow them immediately. + redirected := false for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { totalAttempts++ - if attempt > 0 { + if attempt > 0 && !redirected { if err := internal.Sleep(ctx, c.retryBackoff(attempt)); err != nil { setCmdsErr(cmds, err) - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), totalAttempts, err, nil, 0) - } return err } } - failedCmds := newCmdsMap() - var wg sync.WaitGroup - - for node, cmds := range cmdsMap { - wg.Add(1) - go func(node *clusterNode, cmds []Cmder) { - defer wg.Done() - c.processTxPipelineNode(ctx, node, cmds, failedCmds) - }(node, cmds) + outcome := c.processTxPipelineNode(ctx, node, cmds, asking) + lastErr = outcome.err + redirected = false + switch outcome.kind { + case txSuccess: + return cmdsFirstErr(cmds) + case txRetryMoved: + // Route directly to the authoritative addr from the MOVED; the + // cached slot state may be stale until LazyReload lands. + redirected = true + asking = false + c.state.LazyReload() + if node, err = c.nodes.GetOrCreate(outcome.addr); err != nil { + setCmdsErr(cmds, err) + return err + } + case txRetryAsk: + redirected = true + asking = true + if node, err = c.nodes.GetOrCreate(outcome.addr); err != nil { + setCmdsErr(cmds, err) + return err + } + case txRetryTryAgain, txRetryConn: + // Same node, fresh connection: TRYAGAIN comes from the migrating + // source (still the owner), and a conn failure only needs a new + // connection. Preserve a prior ASKING flag: if we followed an ASK + // to the importing target, the retry must still send ASKING (the + // slot is still importing). ASKING is harmless if the migration + // has since completed, since the flag is only consulted for + // importing slots. + case txFatal: + // Mark every queued-but-never-executed command with the abort + // error; the command that triggered EXECABORT already has its + // own error and keeps it, so callers can tell what went wrong. + abortErr := cmp.Or(outcome.execErr, outcome.err) + for _, cmd := range cmds { + if cmd.Err() == nil { + cmd.SetErr(abortErr) + } + } + return lastErr } - - wg.Wait() - if len(failedCmds.m) == 0 { - break - } - cmdsMap = failedCmds.m - lastErr = cmdsFirstErr(cmds) } - if pipelineOpDurationCallback != nil { - operationDuration := time.Since(operationStart) - finalErr := cmdsFirstErr(cmds) - if finalErr == nil { - finalErr = lastErr - } - pipelineOpDurationCallback(ctx, operationDuration, "MULTI", len(cmds), totalAttempts, finalErr, nil, 0) + if lastErr != nil { + setCmdsErr(cmds, lastErr) } - return cmdsFirstErr(cmds) } // slottedKeyedCommands returns a map of slot to commands taking into account // only commands that have keys. -func (c *ClusterClient) slottedKeyedCommands(ctx context.Context, cmds []Cmder) map[int][]Cmder { +func (c *ClusterClient) slottedKeyedCommands(_ context.Context, cmds []Cmder) map[int][]Cmder { cmdsSlots := map[int][]Cmder{} // Peek once outside the loop, one RLock for the whole batch instead of @@ -1967,151 +2375,255 @@ func (c *ClusterClient) slottedKeyedCommands(ctx context.Context, cmds []Cmder) } func (c *ClusterClient) processTxPipelineNode( - ctx context.Context, node *clusterNode, cmds []Cmder, failedCmds *cmdsMap, -) { - cmds = wrapMultiExec(ctx, cmds) - _ = node.Client.withProcessPipelineHook(ctx, cmds, func(ctx context.Context, cmds []Cmder) error { - cn, err := node.Client.getConn(ctx) - if err != nil { - _ = c.mapCmdsByNode(ctx, failedCmds, cmds) - setCmdsErr(cmds, err) - return err + ctx context.Context, node *clusterNode, cmds []Cmder, asking bool, +) *txOutcome { + wire := wrapMultiExec(ctx, cmds) + if asking { + // ASKING must precede MULTI so the flag stays set for the whole tx. + wire = append([]Cmder{NewCmd(ctx, "asking")}, wire...) + } + + var outcome *txOutcome + // executed guards against a node-level hook short-circuiting (returning + // without calling next) — same treatment as processPipelineNode. + executed := false + chainErr := node.Client.withProcessPipelineHook(ctx, wire, func(ctx context.Context, wire []Cmder) error { + executed = true + // Acquire through the node's dedicated pipeline pool when configured + // (same routing as processPipelineNode); withPipelineConn falls back + // to the main pool otherwise. The inner fn's return value drives the + // connection release exactly like the explicit releaseConn did: + // redis errors keep the conn poolable, unread replies poison it. + entered := false + err := node.Client.withPipelineConn(ctx, func(ctx context.Context, cn *pool.Conn) error { + entered = true + outcome = c.processTxPipelineNodeConn(ctx, node, cn, wire, cmds, asking) + connErr := outcome.err + if isRedisError(outcome.err) { + connErr = nil + } + if outcome.unreadReplies { + connErr = errTxDirtyConn + } + return connErr + }) + if !entered && err != nil { + // Connection acquisition failed — fn never ran. + if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + outcome = &txOutcome{kind: txRetryConn, err: err} + } else { + outcome = &txOutcome{kind: txFatal, err: err} + } } - - var processErr error - defer func() { - node.Client.releaseConn(ctx, cn, processErr) - }() - processErr = c.processTxPipelineNodeConn(ctx, node, cn, cmds, failedCmds) - - return processErr + return err }) + + if !executed && chainErr != nil { + // A node-level hook aborted with an error: surface its verdict. A hook + // that returned nil short-circuited successfully (it served the batch), + // which is legal for plain pipelines too, so it is not turned into a + // fatal outcome (review finding by codex on #3942). + outcome = &txOutcome{kind: txFatal, err: chainErr} + } + if outcome == nil { + outcome = &txOutcome{kind: txFatal, err: fmt.Errorf("redis: tx pipeline produced no outcome")} + } + return outcome } func (c *ClusterClient) processTxPipelineNodeConn( - ctx context.Context, node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds *cmdsMap, -) error { + ctx context.Context, node *clusterNode, cn *pool.Conn, wire []Cmder, cmds []Cmder, asking bool, +) *txOutcome { + // HIMPORT bookkeeping: pending discards and PREPAREs for registered + // fieldsets the transaction references get written ahead of the wire + // batch (before ASKING/MULTI; the session state is visible at EXEC). + injected := node.Client.himportInjectedCmds(ctx, cn, cmds) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { - return writeCmds(wr, cmds) - }); err != nil { - if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { - _ = c.mapCmdsByNode(ctx, failedCmds, cmds) - } - setCmdsErr(cmds, err) - return err - } - - return cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { - statusCmd := cmds[0].(*StatusCmd) - // Trim multi and exec. - trimmedCmds := cmds[1 : len(cmds)-1] - - if err := c.txPipelineReadQueued( - ctx, node, cn, rd, statusCmd, trimmedCmds, failedCmds, - ); err != nil { - setCmdsErr(cmds, err) - - moved, ask, addr := isMovedError(err) - if moved || ask { - return c.cmdsMoved(ctx, trimmedCmds, moved, ask, addr, failedCmds) - } - - return err - } - - return node.Client.pipelineReadCmds(ctx, cn, rd, trimmedCmds) - }) -} - -func (c *ClusterClient) txPipelineReadQueued( - ctx context.Context, - node *clusterNode, - cn *pool.Conn, - rd *proto.Reader, - statusCmd *StatusCmd, - cmds []Cmder, - failedCmds *cmdsMap, -) error { - // Parse queued replies. - // To be sure there are no buffered push notifications, we process them before reading the reply - if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { - // Log the error but don't fail the command execution - // Push notification processing errors shouldn't break normal Redis operations - internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) - } - if err := statusCmd.readReply(rd); err != nil { - return err - } - - for _, cmd := range cmds { - // To be sure there are no buffered push notifications, we process them before reading the reply - if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { - // Log the error but don't fail the command execution - // Push notification processing errors shouldn't break normal Redis operations - internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) - } - err := statusCmd.readReply(rd) - if err != nil { - if c.checkMovedErr(ctx, cmd, err, failedCmds) { - // will be processed later - continue - } - cmd.SetErr(err) - if !isRedisError(err) { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { return err } } + return writeCmds(wr, wire) + }); err != nil { + // Write failure: re-route the whole tx on a fresh connection. + if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + return &txOutcome{kind: txRetryConn, err: err} + } + return &txOutcome{kind: txFatal, err: err} } - // To be sure there are no buffered push notifications, we process them before reading the reply - if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { - // Log the error but don't fail the command execution - // Push notification processing errors shouldn't break normal Redis operations - internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) + var outcome *txOutcome + readErr := cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { + if err := node.Client.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + // Transport error with the tx replies unread; the batch was + // written and may have committed — fatal, discard the conn. + outcome = c.txReadFatal(err) + return nil + } + outcome = c.readTxPipelineReplies(ctx, node, cn, rd, cmds, asking) + if outcome != nil && outcome.kind == txSuccess { + node.Client.himportAfterBatch(cn, injected, cmds) + } + return nil + }) + + if readErr != nil { + // Reader-level failure (deadline setup, nil conn) around the read loop. + // The batch was already written, so the server may have committed; + // surface the error as fatal and discard the suspect connection rather + // than re-executing the transaction. + return c.txReadFatal(readErr) } - // Parse number of replies. + return outcome +} + +// readTxPipelineReplies reads the replies of one MULTI..EXEC unit and +// classifies the outcome. The reply count always matches the number of sent +// commands, so success/redirect paths leave the connection clean; only an early +// MULTI read failure can leave unread replies. +func (c *ClusterClient) readTxPipelineReplies( + ctx context.Context, node *clusterNode, cn *pool.Conn, rd *proto.Reader, cmds []Cmder, asking bool, +) *txOutcome { + scratch := NewStatusCmd(ctx) + + readStatus := func() error { + c.txProcessPush(ctx, node, cn, rd) + return scratch.readReply(rd) + } + + // Optional top-level ASKING reply (+OK, or a retryable error such as -LOADING). + if asking { + if err := readStatus(); err != nil { + return c.txPreQueueErrorOutcome(err, cmds) + } + } + + // MULTI reply (+OK, or an error such as -LOADING during failover). + if err := readStatus(); err != nil { + return c.txPreQueueErrorOutcome(err, cmds) + } + + // Queue replies: +QUEUED, or a redirect / command error that dirties the tx. + var firstRedirect *txRedirect + var firstFatal error + for _, cmd := range cmds { + err := readStatus() + if err == nil { + continue // +QUEUED + } + if !isRedisError(err) { + return c.txReadFatal(err) // IO error + } + if moved, ask, addr := isMovedError(err); moved || ask { + if firstRedirect == nil { + firstRedirect = &txRedirect{moved: moved, ask: ask, addr: addr, err: err} + } + continue + } + if proto.IsTryAgainError(err) { + if firstRedirect == nil { + firstRedirect = &txRedirect{tryAgain: true, err: err} + } + continue + } + // Non-redirect command error (e.g. wrong arity) dirties the tx. + cmd.SetErr(err) + if firstFatal == nil { + firstFatal = err + } + } + + // EXEC reply. ReadLine parses error lines into typed errors, so a non-nil + // err means EXEC returned an error rather than the result array. + c.txProcessPush(ctx, node, cn, rd) line, err := rd.ReadLine() if err != nil { - if err == Nil { - err = TxFailedErr + if !isRedisError(err) { + return c.txReadFatal(err) // IO error } - return err + return c.classifyExecError(err, firstRedirect, firstFatal) } if line[0] != proto.RespArray { - return fmt.Errorf("redis: expected '*', but got line %q", line) + err := fmt.Errorf("redis: unexpected EXEC reply %q", line) + setCmdsErr(cmds, err) + // A non-array aggregate reply may carry an unread payload. + return &txOutcome{kind: txFatal, err: err, unreadReplies: true} } - return nil + // Success: read the N command results. + if err := node.Client.pipelineReadCmds(ctx, cn, rd, cmds); err != nil && !isRedisError(err) { + return c.txReadFatal(err) // IO error mid-results + } + return &txOutcome{kind: txSuccess} } -func (c *ClusterClient) cmdsMoved( - ctx context.Context, cmds []Cmder, - moved, ask bool, - addr string, - failedCmds *cmdsMap, -) error { - node, err := c.nodes.GetOrCreate(addr) - if err != nil { - return err +func (c *ClusterClient) txProcessPush(ctx context.Context, node *clusterNode, cn *pool.Conn, rd *proto.Reader) { + if err := node.Client.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) } +} - if moved { - c.state.LazyReload() - for _, cmd := range cmds { - failedCmds.Add(node, cmd) +// txReadFatal classifies a read-phase IO error. The MULTI..EXEC batch was +// already written, so the server may have committed the transaction; retrying +// would re-execute it, double-applying non-idempotent commands (INCR/APPEND, +// which are not NoRetry). Surface the error as fatal and discard the +// connection, since replies may still be unread on the wire. +func (c *ClusterClient) txReadFatal(err error) *txOutcome { + return &txOutcome{kind: txFatal, err: err, unreadReplies: true} +} + +// txPreQueueErrorOutcome classifies a setup-phase reply error: the top-level +// ASKING reply or the MULTI reply. The transaction body never executes (EXEC +// returns -EXECABORT), so retryable errors such as -LOADING are safe to retry +// on a fresh connection. A failed setup reply still leaves the remaining +// replies on the wire -- the server replies to each following command and to +// EXEC regardless -- so the connection is always discarded. +func (c *ClusterClient) txPreQueueErrorOutcome(err error, cmds []Cmder) *txOutcome { + if !isRedisError(err) { + return c.txReadFatal(err) + } + if shouldRetry(err, true) && !cmdsContainNoRetry(cmds) { + return &txOutcome{kind: txRetryConn, err: err, unreadReplies: true} + } + return &txOutcome{kind: txFatal, err: err, unreadReplies: true} +} + +// classifyExecError turns an EXEC reply error into a retry/fatal outcome. +func (c *ClusterClient) classifyExecError(execErr error, firstRedirect *txRedirect, firstFatal error) *txOutcome { + if moved, ask, addr := isMovedError(execErr); moved || ask { + // Narrow race: the slot moved after every command was queued. + if ask { + return &txOutcome{kind: txRetryAsk, err: execErr, addr: addr} } - return nil + return &txOutcome{kind: txRetryMoved, err: execErr, addr: addr} } - - if ask { - for _, cmd := range cmds { - failedCmds.Add(node, NewCmd(ctx, "asking"), cmd) + if proto.IsTryAgainError(execErr) { + return &txOutcome{kind: txRetryTryAgain, err: execErr} + } + if proto.IsClusterDownError(execErr) { + // Cluster degraded: back off and retry. Replies were fully consumed. + return &txOutcome{kind: txRetryConn, err: execErr} + } + if proto.IsExecAbortError(execErr) { + if firstFatal != nil { + return &txOutcome{kind: txFatal, err: firstFatal, execErr: execErr} } - return nil + if firstRedirect != nil { + switch { + case firstRedirect.moved: + return &txOutcome{kind: txRetryMoved, err: firstRedirect.err, addr: firstRedirect.addr} + case firstRedirect.ask: + return &txOutcome{kind: txRetryAsk, err: firstRedirect.err, addr: firstRedirect.addr} + case firstRedirect.tryAgain: + return &txOutcome{kind: txRetryTryAgain, err: firstRedirect.err} + } + } + return &txOutcome{kind: txFatal, err: execErr, execErr: execErr} } - - return nil + return &txOutcome{kind: txFatal, err: execErr} } func (c *ClusterClient) Watch(ctx context.Context, fn func(*Tx) error, keys ...string) error { @@ -2359,8 +2871,22 @@ func (c *ClusterClient) cmdInfoPeek(name string) *CommandInfo { } func (c *ClusterClient) cmdSlot(cmd Cmder, prefferedSlot int) int { + // Serve/populate the per-command slot cache only on the natural-slot path + // (prefferedSlot == -1). A forced prefferedSlot (retry re-routing) must not be + // cached or served from cache. The cache lets the autopipeline shard router + // and the pipeline-flush router (mapCmdsByNode) share one slot computation + // instead of each recomputing it. + if prefferedSlot == -1 { + if slot, ok := cmd.cachedSlot(); ok { + return slot + } + } info := c.cmdInfoPeek(cmd.Name()) - return c.cmdSlotWithPos(cmd, cmdFirstKeyPosWithInfo(cmd, info), prefferedSlot) + slot := c.cmdSlotWithPos(cmd, cmdFirstKeyPosWithInfo(cmd, info), prefferedSlot) + if prefferedSlot == -1 && slot >= 0 { + cmd.setCachedSlot(slot) + } + return slot } // cmdSlotWithPos computes the cluster slot for cmd given a pre-resolved first key @@ -2520,10 +3046,8 @@ func (c *ClusterClient) NewDynamicResolver() *commandInfoResolver { } func appendIfNotExist[T comparable](vals []T, newVal T) []T { - for _, v := range vals { - if v == newVal { - return vals - } + if slices.Contains(vals, newVal) { + return vals } return append(vals, newVal) } diff --git a/backend/vendor/github.com/redis/go-redis/v9/osscluster_commands.go b/backend/vendor/github.com/redis/go-redis/v9/osscluster_commands.go index b13f8e7e..bc6f60fd 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/osscluster_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/osscluster_commands.go @@ -9,19 +9,19 @@ import ( func (c *ClusterClient) DBSize(ctx context.Context) *IntCmd { cmd := NewIntCmd(ctx, "dbsize") _ = c.withProcessHook(ctx, cmd, func(ctx context.Context, _ Cmder) error { - var size int64 + var size atomic.Int64 err := c.ForEachMaster(ctx, func(ctx context.Context, master *Client) error { n, err := master.DBSize(ctx).Result() if err != nil { return err } - atomic.AddInt64(&size, n) + size.Add(n) return nil }) if err != nil { cmd.SetErr(err) } else { - cmd.val = size + cmd.val = size.Load() } return nil }) diff --git a/backend/vendor/github.com/redis/go-redis/v9/osscluster_router.go b/backend/vendor/github.com/redis/go-redis/v9/osscluster_router.go index 0da29530..b56271d1 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/osscluster_router.go +++ b/backend/vendor/github.com/redis/go-redis/v9/osscluster_router.go @@ -85,7 +85,9 @@ func (c *ClusterClient) executeOnAllNodes(ctx context.Context, cmd Cmder, policy return err } - nodes := append(state.Masters, state.Slaves...) + nodes := make([]*clusterNode, 0, len(state.Masters)+len(state.Slaves)) + nodes = append(nodes, state.Masters...) + nodes = append(nodes, state.Slaves...) if len(nodes) == 0 { return errClusterNoNodes } @@ -494,10 +496,14 @@ func (c *ClusterClient) pickArbitraryNode(ctx context.Context) *clusterNode { return nil } - allNodes := append(state.Masters, state.Slaves...) - - idx := c.opt.ShardPicker.Next(len(allNodes)) - return allNodes[idx] + // Index into masters+slaves without materializing a combined slice. + // append(state.Masters, state.Slaves...) writes into the shared snapshot's + // spare capacity and races other routers, so pick directly. + idx := c.opt.ShardPicker.Next(len(state.Masters) + len(state.Slaves)) + if idx < len(state.Masters) { + return state.Masters[idx] + } + return state.Slaves[idx-len(state.Masters)] } // hasKeys checks if a command operates on keys diff --git a/backend/vendor/github.com/redis/go-redis/v9/probabilistic.go b/backend/vendor/github.com/redis/go-redis/v9/probabilistic.go index ee67911e..f4c40262 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/probabilistic.go +++ b/backend/vendor/github.com/redis/go-redis/v9/probabilistic.go @@ -233,6 +233,7 @@ func newScanDumpCmd(ctx context.Context, args ...interface{}) *ScanDumpCmd { } func (cmd *ScanDumpCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -241,10 +242,12 @@ func (cmd *ScanDumpCmd) SetVal(val ScanDump) { } func (cmd *ScanDumpCmd) Result() (ScanDump, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *ScanDumpCmd) Val() ScanDump { + cmd.await() return cmd.val } @@ -316,14 +319,17 @@ func (cmd *BFInfoCmd) SetVal(val BFInfo) { } func (cmd *BFInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *BFInfoCmd) Val() BFInfo { + cmd.await() return cmd.val } func (cmd *BFInfoCmd) Result() (BFInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -653,14 +659,17 @@ func (cmd *CFInfoCmd) SetVal(val CFInfo) { } func (cmd *CFInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *CFInfoCmd) Val() CFInfo { + cmd.await() return cmd.val } func (cmd *CFInfoCmd) Result() (CFInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -823,14 +832,17 @@ func (cmd *CMSInfoCmd) SetVal(val CMSInfo) { } func (cmd *CMSInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *CMSInfoCmd) Val() CMSInfo { + cmd.await() return cmd.val } func (cmd *CMSInfoCmd) Result() (CMSInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -1024,14 +1036,17 @@ func (cmd *TopKInfoCmd) SetVal(val TopKInfo) { } func (cmd *TopKInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *TopKInfoCmd) Val() TopKInfo { + cmd.await() return cmd.val } func (cmd *TopKInfoCmd) Result() (TopKInfo, error) { + cmd.await() return cmd.val, cmd.err } @@ -1279,14 +1294,17 @@ func (cmd *TDigestInfoCmd) SetVal(val TDigestInfo) { } func (cmd *TDigestInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } func (cmd *TDigestInfoCmd) Val() TDigestInfo { + cmd.await() return cmd.val } func (cmd *TDigestInfoCmd) Result() (TDigestInfo, error) { + cmd.await() return cmd.val, cmd.err } diff --git a/backend/vendor/github.com/redis/go-redis/v9/pubsub_commands.go b/backend/vendor/github.com/redis/go-redis/v9/pubsub_commands.go index ccc0ed52..b0857b9c 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/pubsub_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/pubsub_commands.go @@ -20,8 +20,15 @@ type PubSubCmdable interface { func (c cmdable) Publish(ctx context.Context, channel string, message interface{}) *IntCmd { cmd := NewIntCmd(ctx, "publish", channel, message) _ = c(ctx, cmd) - // Record PubSub message sent (if command succeeded) - if cmd.Err() == nil { + // Record PubSub message sent (if command succeeded). Gated on the result + // being readable WITHOUT blocking: on the deferred autopipeline face the + // call above only enqueues, so reading the outcome here would await the + // batch and turn a fire-and-forget publish into a blocking call — i.e. + // enabling telemetry would change the async call shape (review finding by + // codex on #3942). The metric is therefore skipped for a submission that + // has not executed yet; recording it from the execution path instead is a + // follow-up in the OTel wiring, not something the command wrapper can do. + if otel.Enabled() && cmd.resultReady() && cmd.rawErr() == nil { otel.RecordPubSubMessage(ctx, nil, "sent", channel, false) } return cmd @@ -30,8 +37,9 @@ func (c cmdable) Publish(ctx context.Context, channel string, message interface{ func (c cmdable) SPublish(ctx context.Context, channel string, message interface{}) *IntCmd { cmd := NewIntCmd(ctx, "spublish", channel, message) _ = c(ctx, cmd) - // Record PubSub message sent (if command succeeded) - if cmd.Err() == nil { + // Record PubSub message sent (if command succeeded). See Publish for why + // this is gated on the result being readable without blocking. + if otel.Enabled() && cmd.resultReady() && cmd.rawErr() == nil { otel.RecordPubSubMessage(ctx, nil, "sent", channel, true) } return cmd diff --git a/backend/vendor/github.com/redis/go-redis/v9/push/processor.go b/backend/vendor/github.com/redis/go-redis/v9/push/processor.go index b8112ddc..264d779d 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/push/processor.go +++ b/backend/vendor/github.com/redis/go-redis/v9/push/processor.go @@ -2,6 +2,7 @@ package push import ( "context" + "errors" "github.com/redis/go-redis/v9/internal" "github.com/redis/go-redis/v9/internal/proto" @@ -26,6 +27,15 @@ type Processor struct { registry *Registry } +type timeoutError interface { + Timeout() bool +} + +func isTimeoutError(err error) bool { + var timeoutErr timeoutError + return errors.As(err, &timeoutErr) && timeoutErr.Timeout() +} + // NewProcessor creates a new push notification processor func NewProcessor() *Processor { return &Processor{ @@ -52,17 +62,59 @@ func (p *Processor) UnregisterHandler(pushNotificationName string) error { // This method should be called by the client in WithReader before reading the reply // It will try to read from the socket and if it is empty - it may block. func (p *Processor) ProcessPendingNotifications(ctx context.Context, handlerCtx NotificationHandlerContext, rd *proto.Reader) error { + return p.processPendingNotifications(ctx, handlerCtx, rd, false) +} + +// ProcessPendingNotificationsBuffered processes one pending push notification +// and then continues only through frames already buffered by that read. It is +// used by callers that have already established socket readiness and must not +// wait for another frame after draining the current batch. +func (p *Processor) ProcessPendingNotificationsBuffered( + ctx context.Context, handlerCtx NotificationHandlerContext, rd *proto.Reader, +) error { + return p.processPendingNotifications(ctx, handlerCtx, rd, true) +} + +func (p *Processor) processPendingNotifications( + ctx context.Context, + handlerCtx NotificationHandlerContext, + rd *proto.Reader, + bufferedContinuation bool, +) error { if rd == nil { return nil } - for { + processed := false + for !bufferedContinuation || !processed || rd.Buffered() > 0 { // Check if there's data available to read - replyType, err := rd.PeekReplyType() - if err != nil { - // No more data available or error reading - // if timeout, it will be handled by the caller - break + var replyType byte + if bufferedContinuation { + for { + b, err := rd.Peek(1) + if err != nil { + if isTimeoutError(err) { + return nil + } + return err + } + replyType = b[0] + if replyType != proto.RespAttr { + break + } + // Unlike Peek, DiscardNext consumes bytes. Any error here is + // fatal because the reader may be left mid-frame. + if err := rd.DiscardNext(); err != nil { + return err + } + } + } else { + var err error + replyType, err = rd.PeekReplyType() + if err != nil { + // No more data available or error reading. + break + } } // Only process push notifications (arrays starting with >) @@ -73,19 +125,31 @@ func (p *Processor) ProcessPendingNotifications(ctx context.Context, handlerCtx // see if we should skip this notification notificationName, err := rd.PeekPushNotificationName() if err != nil { + // Name too long to peek: consume & dispatch below rather than leave the + // frame at the buffer head (which would stall and desync the next reply). + if !errors.Is(err, proto.ErrPushNotificationNameTooLong) { + if bufferedContinuation { + if isTimeoutError(err) { + return nil + } + return err + } + break + } + } else if willHandleNotificationInClient(notificationName) { break } - if willHandleNotificationInClient(notificationName) { - break - } - - // Read the push notification + // Surface a ReadReply error (unlike the boundary peek errors above, + // which consumed nothing): it happens mid-frame after bytes are + // consumed, so the conn is desynced and the CSC drainer must remove it. + // Normal reply-read callers log-and-ignore this and let their own read fail. reply, err := rd.ReadReply() if err != nil { internal.Logger.Printf(ctx, "push: error reading push notification: %v", err) - break + return err } + processed = true // Convert to slice of interfaces notification, ok := reply.([]interface{}) @@ -165,10 +229,12 @@ func (v *VoidProcessor) ProcessPendingNotifications(_ context.Context, handlerCt // see if we should skip this notification notificationName, err := rd.PeekPushNotificationName() if err != nil { - break - } - - if willHandleNotificationInClient(notificationName) { + // Name too long to peek: still consume the frame below so it isn't + // misread as a reply. + if !errors.Is(err, proto.ErrPushNotificationNameTooLong) { + break + } + } else if willHandleNotificationInClient(notificationName) { break } diff --git a/backend/vendor/github.com/redis/go-redis/v9/redis.go b/backend/vendor/github.com/redis/go-redis/v9/redis.go index 3bb759c3..8883a1ad 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/redis.go +++ b/backend/vendor/github.com/redis/go-redis/v9/redis.go @@ -1,6 +1,7 @@ package redis import ( + "bytes" "context" "errors" "fmt" @@ -64,17 +65,61 @@ type ( ) type hooksMixin struct { - hooksMu *sync.RWMutex + // hooksMu serializes writers (AddHook); readers never take it. + hooksMu *sync.Mutex + // state holds the immutable hook snapshot. Readers Load it lock-free; + // writers publish a replacement copy-on-write under hooksMu. + state *atomic.Pointer[hooksState] +} +// hooksState is an immutable snapshot of the hook configuration. Once stored +// in hooksMixin.state it is never mutated; AddHook builds a fresh copy. +type hooksState struct { slice []Hook initial hooks current hooks } +// rebuild recomputes current from initial + slice. It mutates the receiver, so +// it must only run on a state that has not yet been published. +func (s *hooksState) rebuild() { + s.initial.setDefaults() + + s.current.dial = s.initial.dial + s.current.process = s.initial.process + s.current.pipeline = s.initial.pipeline + s.current.txPipeline = s.initial.txPipeline + + for i := len(s.slice) - 1; i >= 0; i-- { + if wrapped := s.slice[i].DialHook(s.current.dial); wrapped != nil { + s.current.dial = wrapped + } + if wrapped := s.slice[i].ProcessHook(s.current.process); wrapped != nil { + s.current.process = wrapped + } + if wrapped := s.slice[i].ProcessPipelineHook(s.current.pipeline); wrapped != nil { + s.current.pipeline = wrapped + } + if wrapped := s.slice[i].ProcessPipelineHook(s.current.txPipeline); wrapped != nil { + s.current.txPipeline = wrapped + } + } +} + func (hs *hooksMixin) initHooks(hooks hooks) { - hs.hooksMu = new(sync.RWMutex) - hs.initial = hooks - hs.chain() + var slice []Hook + if hs.state != nil { + if old := hs.state.Load(); old != nil { + slice = old.slice + } + } + + hs.hooksMu = new(sync.Mutex) + hs.state = new(atomic.Pointer[hooksState]) + + state := &hooksState{slice: slice, initial: hooks} + state.rebuild() + hs.state.Store(state) } type hooks struct { @@ -136,51 +181,42 @@ func (h *hooks) setDefaults() { // Please note: "next(ctx, cmd)" is very important, it will call the next hook, // if "next(ctx, cmd)" is not executed, the redis command will not be executed. func (hs *hooksMixin) AddHook(hook Hook) { - hs.slice = append(hs.slice, hook) - hs.chain() -} - -func (hs *hooksMixin) chain() { - hs.initial.setDefaults() - hs.hooksMu.Lock() defer hs.hooksMu.Unlock() - hs.current.dial = hs.initial.dial - hs.current.process = hs.initial.process - hs.current.pipeline = hs.initial.pipeline - hs.current.txPipeline = hs.initial.txPipeline - - for i := len(hs.slice) - 1; i >= 0; i-- { - if wrapped := hs.slice[i].DialHook(hs.current.dial); wrapped != nil { - hs.current.dial = wrapped - } - if wrapped := hs.slice[i].ProcessHook(hs.current.process); wrapped != nil { - hs.current.process = wrapped - } - if wrapped := hs.slice[i].ProcessPipelineHook(hs.current.pipeline); wrapped != nil { - hs.current.pipeline = wrapped - } - if wrapped := hs.slice[i].ProcessPipelineHook(hs.current.txPipeline); wrapped != nil { - hs.current.txPipeline = wrapped - } + old := hs.state.Load() + state := &hooksState{ + slice: make([]Hook, len(old.slice)+1), + initial: old.initial, } + copy(state.slice, old.slice) + state.slice[len(old.slice)] = hook + state.rebuild() + + hs.state.Store(state) } func (hs *hooksMixin) clone() hooksMixin { - hs.hooksMu.Lock() - defer hs.hooksMu.Unlock() + old := hs.state.Load() + l := len(old.slice) + state := &hooksState{ + slice: old.slice[:l:l], + initial: old.initial, + current: old.current, + } - clone := *hs - l := len(clone.slice) - clone.slice = clone.slice[:l:l] - clone.hooksMu = new(sync.RWMutex) + clone := hooksMixin{ + hooksMu: new(sync.Mutex), + state: new(atomic.Pointer[hooksState]), + } + clone.state.Store(state) return clone } func (hs *hooksMixin) withProcessHook(ctx context.Context, cmd Cmder, hook ProcessHook) error { - for i := len(hs.slice) - 1; i >= 0; i-- { - if wrapped := hs.slice[i].ProcessHook(hook); wrapped != nil { + slice := hs.state.Load().slice + for i := len(slice) - 1; i >= 0; i-- { + if wrapped := slice[i].ProcessHook(hook); wrapped != nil { hook = wrapped } } @@ -190,8 +226,9 @@ func (hs *hooksMixin) withProcessHook(ctx context.Context, cmd Cmder, hook Proce func (hs *hooksMixin) withProcessPipelineHook( ctx context.Context, cmds []Cmder, hook ProcessPipelineHook, ) error { - for i := len(hs.slice) - 1; i >= 0; i-- { - if wrapped := hs.slice[i].ProcessPipelineHook(hook); wrapped != nil { + slice := hs.state.Load().slice + for i := len(slice) - 1; i >= 0; i-- { + if wrapped := slice[i].ProcessPipelineHook(hook); wrapped != nil { hook = wrapped } } @@ -199,26 +236,26 @@ func (hs *hooksMixin) withProcessPipelineHook( } func (hs *hooksMixin) dialHook(ctx context.Context, network, addr string) (net.Conn, error) { - // Access to hs.current is guarded by a read-only lock since it may be mutated by AddHook(...) - // while this dialer is concurrently accessed by the background connection pool population - // routine when MinIdleConns > 0. - hs.hooksMu.RLock() - current := hs.current - hs.hooksMu.RUnlock() + return hs.state.Load().current.dial(ctx, network, addr) +} - return current.dial(ctx, network, addr) +// hookCount reports how many user hooks are installed. The autopipeliner +// arms its await() self-deadlock guard only when hooks exist, keeping the +// guard a single atomic load on hook-free clients. +func (hs *hooksMixin) hookCount() int { + return len(hs.state.Load().slice) } func (hs *hooksMixin) processHook(ctx context.Context, cmd Cmder) error { - return hs.current.process(ctx, cmd) + return hs.state.Load().current.process(ctx, cmd) } func (hs *hooksMixin) processPipelineHook(ctx context.Context, cmds []Cmder) error { - return hs.current.pipeline(ctx, cmds) + return hs.state.Load().current.pipeline(ctx, cmds) } func (hs *hooksMixin) processTxPipelineHook(ctx context.Context, cmds []Cmder) error { - return hs.current.txPipeline(ctx, cmds) + return hs.state.Load().current.txPipeline(ctx, cmds) } //------------------------------------------------------------------------------ @@ -314,10 +351,26 @@ func (h *onCloseHooks) run() error { } type baseClient struct { + // apClosed flips when the shared pools begin closing; every wrapper and + // every clone SHARING those pools refuses to build a new autopipeliner + // from then on. A pointer: withTimeout/clone copy it, so the flag is one + // per pool-set, not one per wrapper. See baseClient.Close. + apClosed *atomic.Bool + opt *Options optLock sync.RWMutex connPool pool.Pooler pubSubPool *pool.PubSubPool + // pipelinePool is an optional separate connection pool for pipelining + // operations, used when PipelineReadBufferSize/PipelineWriteBufferSize is + // set so pipelines can use large buffers without bloating the main pool. + // nil means pipelines use connPool. + pipelinePool pool.Pooler + // pipelinePoolName is the pool name assigned to pipelinePool's connections + // (pool.Conn.PoolName()). It lets poolForConn route a connection back to the + // pool that owns it — e.g. so streaming-credentials re-auth closes/accounts a + // failed pipeline connection against pipelinePool, not connPool. + pipelinePoolName string hooksMixin // onClose holds named callbacks invoked when the client is closed. @@ -337,6 +390,51 @@ type baseClient struct { // streamingCredentialsManager is used to manage streaming credentials streamingCredentialsManager *streaming.Manager + + // himport is the client-side registry of HIMPORT fieldsets, used to + // lazily replay HIMPORT PREPARE onto pooled connections (see himport.go). + // Shared by clones and by Conn instances derived from the same pool. + himport *himportRegistry + + // csc is the shared client-side cache; nil when CSC is disabled. + csc Cache + + // cscKeyPrefix namespaces a shared cache by DB and fixed authentication + // identity. It is computed once during attachment and copied with the cache. + cscKeyPrefix string + + // allowClientTracking exempts a client from the CLIENT TRACKING guard (see + // process and generalProcessPipeline). Set only on initConn's internal conn + // wrapper, whose init pipeline legitimately issues CLIENT TRACKING ON; + // never set on user-visible clients. + allowClientTracking bool + + // The following are OWNER-ONLY and NOT copied by clone(): derived clients + // (Conn/WithTimeout) share the cache but must not stop the owner's + // goroutines or flush its cache on their own Close. + + // cscOwnsCache is true only when this client constructed its LocalCache (not + // an injected/shared one); it gates the defensive flush on drainer stop. + cscOwnsCache bool + + // cscDrainHandle is the background drainer handle (nil when none). Held + // on the client, not a global registry, so an un-Closed client stays + // GC-collectible and a runtime.AddCleanup net can stop the goroutine. Its + // presence also identifies the owner (the only client that runs the drainer + // and thus the one that deregisters cscPoolHook). + cscDrainHandle *cscDrainHandle + + // cscPoolHook is the evict-on-remove pool hook (nil when CSC is off). Unlike + // the owner-only fields above it IS copied by clone(): a clone reads it in + // processCached to attribute fetches to the shared hook. Only the owner (the + // one with cscDrainHandle) deregisters it when the drainer exits. + cscPoolHook pool.PoolHook + + // cscActive is allocated only after CSC attaches successfully and becomes + // false once the drainer stops (owner Close, GC cleanup, or damping). It is + // shared with derived clients so they initialize borrowed pool connections + // with tracking only while the parent's CSC is actually operational. + cscActive *atomic.Bool } func (c *baseClient) clone() *baseClient { @@ -345,13 +443,24 @@ func (c *baseClient) clone() *baseClient { c.maintNotificationsManagerLock.RUnlock() clone := &baseClient{ + apClosed: c.apClosed, opt: c.opt, connPool: c.connPool, + pipelinePool: c.pipelinePool, + pipelinePoolName: c.pipelinePoolName, pubSubPool: c.pubSubPool, onClose: c.onClose, pushProcessor: c.pushProcessor, maintNotificationsManager: maintNotificationsManager, streamingCredentialsManager: c.streamingCredentialsManager, + himport: c.himport, + csc: c.csc, + // cscPoolHook and cscActive travel with the cache (read in processCached); + // the owner-only fields — cscDrainHandle, cscOwnsCache — do not, so a clone's + // Close never tears down the owner's resources. + cscPoolHook: c.cscPoolHook, + cscActive: c.cscActive, + cscKeyPrefix: c.cscKeyPrefix, } return clone } @@ -405,16 +514,30 @@ func (c *baseClient) _getConn(ctx context.Context) (*pool.Conn, error) { return nil, err } + if err := c.initPooledConn(ctx, c.connPool, cn); err != nil { + return nil, err + } + + return cn, nil +} + +// initPooledConn brings a conn freshly obtained from p to a usable state: it +// runs the connection handshake if needed, records the connection-create-time +// metric, and re-acquires the conn after initConn parks it IDLE. On failure +// the conn is Removed from p (never leaked) and the error is unwrapped to the +// caller-visible cause. Shared by the main-pool path (_getConn) and the +// dedicated pipeline-pool path (withPipelineConn) so the two cannot drift. +func (c *baseClient) initPooledConn(ctx context.Context, p pool.Pooler, cn *pool.Conn) error { if cn.IsInited() { - return cn, nil + return nil } if err := c.initConn(ctx, cn); err != nil { - c.connPool.Remove(ctx, cn, err) - if err := errors.Unwrap(err); err != nil { - return nil, err + p.Remove(ctx, cn, err) + if unwrapped := errors.Unwrap(err); unwrapped != nil { + return unwrapped } - return nil, err + return err } if dialStartNs := cn.GetDialStartNs(); dialStartNs > 0 { @@ -427,10 +550,25 @@ func (c *baseClient) _getConn(ctx context.Context) (*pool.Conn, error) { // initConn will transition to IDLE state, so we need to acquire it // before returning it to the user. if !cn.TryAcquire() { - return nil, fmt.Errorf("redis: connection is not usable") + err := fmt.Errorf("redis: connection is not usable") + // Remove rather than abandon: an unacquirable conn left outside the + // pool's accounting would leak its slot. + p.Remove(ctx, cn, err) + return err } - return cn, nil + return nil +} + +// poolForConn returns the pool that owns cn — the dedicated pipeline pool when +// cn was dialed there, otherwise the main pool. Re-auth close/accounting must +// target the owning pool so a failed pipeline connection is removed from the +// pipeline pool's books, not the main pool's. +func (c *baseClient) poolForConn(cn *pool.Conn) pool.Pooler { + if c.pipelinePool != nil && c.pipelinePoolName != "" && cn.PoolName() == c.pipelinePoolName { + return c.pipelinePool + } + return c.connPool } func (c *baseClient) reAuthConnection() func(poolCn *pool.Conn, credentials auth.Credentials) error { @@ -441,10 +579,11 @@ func (c *baseClient) reAuthConnection() func(poolCn *pool.Conn, credentials auth // Use background context - timeout is handled by ReadTimeout in WithReader/WithWriter ctx := context.Background() - connPool := pool.NewSingleConnPool(c.connPool, poolCn) + connPool := pool.NewSingleConnPool(c.poolForConn(poolCn), poolCn) - // Pass hooks so that reauth commands are recorded/traced - cn := newConn(c.opt, connPool, &c.hooksMixin) + // Pass hooks so that reauth commands are recorded/traced; share the + // HIMPORT registry for the same reason as in initConn. + cn := newConn(c.opt, connPool, &c.hooksMixin, c.himport) if username != "" { err = cn.AuthACL(ctx, username, password).Err() @@ -455,6 +594,7 @@ func (c *baseClient) reAuthConnection() func(poolCn *pool.Conn, credentials auth return err } } + func (c *baseClient) onAuthenticationErr() func(poolCn *pool.Conn, err error) { return func(poolCn *pool.Conn, err error) { if err != nil { @@ -464,7 +604,7 @@ func (c *baseClient) onAuthenticationErr() func(poolCn *pool.Conn, err error) { // waits for IDLE state before transitioning to UNUSABLE for re-auth). // From metrics perspective, the connection was never "used" by a client. // Note: Using context.Background() as this callback doesn't have access to caller's context. - err := c.connPool.CloseConn(context.Background(), poolCn, pool.CloseReasonAuthError, pool.MetricStateIdle) + err := c.poolForConn(poolCn).CloseConn(context.Background(), poolCn, pool.CloseReasonAuthError, pool.MetricStateIdle) if err != nil { internal.Logger.Printf(context.Background(), "redis: failed to close connection: %v", err) // try to close the network connection directly @@ -480,6 +620,25 @@ func (c *baseClient) onAuthenticationErr() func(poolCn *pool.Conn, err error) { } } +// resolveCredentials returns the username/password to authenticate with, using +// the non-streaming credential sources in precedence order: +// CredentialsProviderContext, then CredentialsProvider, then the static +// Username/Password fields. The StreamingCredentialsProvider path is handled +// separately by initConn (it requires per-connection listener wiring) and is +// intentionally not covered here. Returns empty strings when no credentials +// are configured. +func (opt *Options) resolveCredentials(ctx context.Context) (username, password string, err error) { + switch { + case opt.CredentialsProviderContext != nil: + return opt.CredentialsProviderContext(ctx) + case opt.CredentialsProvider != nil: + username, password = opt.CredentialsProvider() + case opt.Username != "" || opt.Password != "": + username, password = opt.Username, opt.Password + } + return username, password, nil +} + func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { // This function is called in two scenarios: // 1. First-time init: Connection is in CREATED state (from pool.Get()) @@ -558,7 +717,22 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { // If we fail, we must transition to CLOSED var initErr error connPool := pool.NewSingleConnPool(c.connPool, cn) - conn := newConn(c.opt, connPool, &c.hooksMixin) + // The handshake Conn (handed to OnConnect) must share the client's + // HIMPORT registry: a private registry restarts versions at 1, so an + // OnConnect prepare would mark the pooled connection with a version + // number that collides with the client registry's and silently skips + // the replay of a different fieldset definition. + conn := newConn(c.opt, connPool, &c.hooksMixin, c.himport) + // The internal wrapper does not serve cached reads, but it needs the + // successful-attachment signal both to issue CLIENT TRACKING during init + // and to guard the user-visible OnConnect callback below. + conn.baseClient.cscActive = c.cscActive + // This internal conn's init pipeline issues CLIENT TRACKING ON itself; + // exempt it from the guard that blocks user-issued CLIENT TRACKING. Setting + // the field after newConn is safe: initHooks bound the pipeline hook as a + // method value on the addressable baseClient, so the guard reads the + // updated field. + conn.baseClient.allowClientTracking = true username, password := "", "" if c.opt.StreamingCredentialsProvider != nil { @@ -598,16 +772,12 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { cn.SetOnClose(unsubscribeFromCredentialsProvider) username, password = credentials.BasicAuth() - } else if c.opt.CredentialsProviderContext != nil { - username, password, initErr = c.opt.CredentialsProviderContext(ctx) + } else { + username, password, initErr = c.opt.resolveCredentials(ctx) if initErr != nil { cn.GetStateMachine().Transition(pool.StateClosed) - return fmt.Errorf("failed to get credentials from context provider: %w", initErr) + return fmt.Errorf("failed to resolve credentials: %w", initErr) } - } else if c.opt.CredentialsProvider != nil { - username, password = c.opt.CredentialsProvider() - } else if c.opt.Username != "" || c.opt.Password != "" { - username, password = c.opt.Username, c.opt.Password } // for redis-server versions that do not support the HELLO command, @@ -616,6 +786,10 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { // falls back to RESP2 regardless of c.opt.Protocol, and features that // require RESP3 (e.g. maintenance notifications) must be skipped. helloOK := false + // For redis-server versions that do not support HELLO, RESP2 continues to + // be used. Remember that negotiated fallback: configured Protocol remains 3, + // but CSC must not serve without RESP3 invalidations. + helloFallbackToRESP2 := false if initErr = conn.Hello(ctx, c.opt.Protocol, username, password, c.opt.ClientName).Err(); initErr == nil { // Authentication successful with HELLO command helloOK = true @@ -629,20 +803,40 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { // difficult to rely on error strings to determine all results. cn.GetStateMachine().Transition(pool.StateClosed) return initErr - } else if password != "" { - // Try legacy AUTH command if HELLO failed - if username != "" { - initErr = conn.AuthACL(ctx, username, password).Err() - } else { - initErr = conn.Auth(ctx, password).Err() - } - if initErr != nil { - cn.GetStateMachine().Transition(pool.StateClosed) - return fmt.Errorf("failed to authenticate: %w", initErr) + } else { + helloFallbackToRESP2 = c.opt.Protocol == 3 + if password != "" { + // Try legacy AUTH command if HELLO failed. + if username != "" { + initErr = conn.AuthACL(ctx, username, password).Err() + } else { + initErr = conn.Auth(ctx, password).Err() + } + if initErr != nil { + cn.GetStateMachine().Transition(pool.StateClosed) + return fmt.Errorf("failed to authenticate: %w", initErr) + } } } + if helloFallbackToRESP2 { + c.disableCSCServing(ctx, "HELLO 3 was rejected and the connection negotiated RESP2") + } - _, initErr = conn.Pipelined(ctx, func(pipe Pipeliner) error { + // trackingEnabled reports whether THIS pool connection must issue + // CLIENT TRACKING ON during init. True when CSC (SharedTracking) is enabled: + // the shared cache is fed by per-connection tracking + the background + // drainer. Once CSC serving stops (owner Close, GC cleanup, or drainer + // damping), new and re-inited conns skip tracking — nothing consumes the + // pushes into the cache anymore. + trackingEnabled := !helloFallbackToRESP2 && !cn.IsPubSub() && c.cscTrackingRequested() + if trackingEnabled && c.cscConnInitGen(cn.GetID()) == 0 { + // First initialization establishes generation 1. Reinitialization + // already bumped and evicted through onCscReinit before replacing the + // socket, so it must not bump a second time here. + c.cscEvictOwnedEntries(cn.GetID()) + } + var trackingCmd *StatusCmd + initCmds, initErr := conn.Pipelined(ctx, func(pipe Pipeliner) error { if c.opt.DB > 0 { pipe.Select(ctx, c.opt.DB) } @@ -655,13 +849,56 @@ func (c *baseClient) initConn(ctx context.Context, cn *pool.Conn) error { pipe.ClientSetName(ctx, c.opt.ClientName) } + if trackingEnabled { + // Must run before any cacheable command is issued on this conn. + trackingCmd = pipe.ClientTrackingOn(ctx, nil) + } + return nil }) + // The exemption is init-only. OnConnect is user code and must go through + // the same CSC connection-state guard as every other public command path. + conn.baseClient.allowClientTracking = false + trackingRejected := trackingCmd != nil && isRedisError(trackingCmd.Err()) + for _, cmd := range initCmds { + if cmd != trackingCmd && cmd.Err() != nil { + trackingRejected = false + break + } + } + if trackingRejected { + // A server-side rejection means tracking is unavailable, but the + // connection and the preceding init commands are still usable. Disable + // CSC globally and continue without caching. Transport and protocol + // failures still take the normal connection-failure path below. + c.disableCSCServing(ctx, fmt.Sprintf("CLIENT TRACKING ON was rejected: %v", trackingCmd.Err())) + c.cscForgetConn(cn.GetID()) + trackingEnabled = false + initErr = nil + } if initErr != nil { + if trackingEnabled { + // cscEvictOwnedEntries above bumped this conn's init generation; a + // failed init never serves, and the pubsub path has no OnRemove + // hook (and the close hook below is not yet installed), so drop + // the entry here to keep the map bounded to live conns. + c.cscForgetConn(cn.GetID()) + } cn.GetStateMachine().Transition(pool.StateClosed) return fmt.Errorf("failed to initialize connection options: %w", initErr) } + if trackingEnabled { + // Evict this conn's entries on any close (incl. the ConnMaxLifetime/idle + // path that bypasses the OnRemove hook), since the server drops its + // tracking table on close. + c.cscInstallConnCloseHook(cn) + // A handoff replaces the socket before initConn runs. Bump and evict at + // the pre-swap boundary so fulfillCached cannot publish an old-socket + // reply during that gap. + c.cscInstallConnReinitHook(cn) + } + // Enable maintnotifications if maintnotifications are configured c.optLock.RLock() maintNotifEnabled := c.opt.MaintNotificationsConfig != nil && c.opt.MaintNotificationsConfig.Mode != maintnotifications.ModeDisabled @@ -799,16 +1036,38 @@ func (c *baseClient) releaseConn(ctx context.Context, cn *pool.Conn, err error) if c.opt.Limiter != nil { c.opt.Limiter.ReportResult(err) } + c.releaseConnToPool(ctx, c.connPool, cn, err) +} +// releaseConnToPool returns a conn to p after a command or pipeline ran on +// it: bad conns are Removed, pending push notifications are drained (a +// mid-frame drain failure also Removes — the reply stream may be +// desynchronized), and a client-side-cache post-read probe is requested when +// tracking is on. Limiter accounting stays with the callers, whose shapes +// differ. Shared by releaseConn and withPipelineConn so the two cannot drift. +func (c *baseClient) releaseConnToPool(ctx context.Context, p pool.Pooler, cn *pool.Conn, err error) { if isBadConn(err, false, c.opt.Addr) { - c.connPool.Remove(ctx, cn, err) - } else { - // process any pending push notifications before returning the connection to the pool - if err := c.processPushNotifications(ctx, cn); err != nil { - internal.Logger.Printf(ctx, "push: error processing pending notifications before releasing connection: %v", err) - } - c.connPool.Put(ctx, cn) + p.Remove(ctx, cn, err) + return } + // process any pending push notifications before returning the connection to the pool + if err := c.processPushNotifications(ctx, cn); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before releasing connection: %v", err) + if isBadConn(err, false, c.opt.Addr) { + // A mid-frame read failure may leave the reply stream + // desynchronized, so the connection cannot be reused. + p.Remove(ctx, cn, err) + return + } + } + if c.cscTrackingRequested() { + // A TLS-like wrapper can retain decrypted bytes after the command + // reply even when its raw socket is empty. Ask the background + // drainer for one bounded post-read probe before relying on raw + // socket peeks again. + cn.MarkCscReadPending() + } + p.Put(ctx, cn) } func (c *baseClient) withConn( @@ -829,39 +1088,158 @@ func (c *baseClient) withConn( return fnErr } +// withPipelineConn executes fn with a connection from the pipeline pool when +// one is configured (PipelineReadBufferSize/PipelineWriteBufferSize set), +// otherwise it falls back to the regular pool via withConn. +// withPipelineConn is withConn/releaseConn for the DEDICATED pipeline pool. +// Conn preparation and release go through the shared pool-parameterized +// helpers (initPooledConn, releaseConnToPool) — the paths used to mirror each +// other by hand and drifted three times (a Limiter-ordering divergence, a +// missed drain-error removal, a missed client-side-cache probe), so only the +// Limiter shape is allowed to live here. +func (c *baseClient) withPipelineConn( + ctx context.Context, fn func(context.Context, *pool.Conn) error, +) (retErr error) { + // Use pipeline pool if available, otherwise fall back to regular pool. + if c.pipelinePool == nil { + return c.withConn(ctx, fn) + } + + // Honor the Limiter on the dedicated pipeline-pool path too, mirroring + // getConn/releaseConn: Allow() before acquiring and ReportResult() on every + // exit (including the early init/re-acquire failures below). Without this, + // enabling the pipeline pool would silently bypass throttling and failure + // reporting for callers that set a Limiter. + if c.opt.Limiter != nil { + if err := c.opt.Limiter.Allow(); err != nil { + return err + } + } + + // One deferred exit for both concerns, because their ORDER is part of the + // contract: releaseConn reports the result BEFORE the connection becomes + // available again, so a limiter or circuit breaker observes the failure + // before it can admit the next operation. Two separate defers would run + // LIFO and release first, letting another pipelined operation through + // against a breaker that has not seen the failure yet (review finding by + // codex on #3942). cn is nil on the acquire/init failure paths, which still + // must report. + var cn *pool.Conn + var fnErr error + defer func() { + if c.opt.Limiter != nil { + c.opt.Limiter.ReportResult(retErr) + } + if cn != nil { + c.releaseConnToPool(ctx, c.pipelinePool, cn, fnErr) + } + }() + + cn, retErr = c.pipelinePool.Get(ctx) + if retErr != nil { + cn = nil // nothing acquired: no release, but still report above + return retErr + } + + if err := c.initPooledConn(ctx, c.pipelinePool, cn); err != nil { + // initPooledConn already removed the conn from the pool on failure. + cn = nil + retErr = err + return retErr + } + + fnErr = fn(ctx, cn) + retErr = fnErr + return retErr +} + func (c *baseClient) dial(ctx context.Context, network, addr string) (net.Conn, error) { return c.opt.Dialer(ctx, network, addr) } -func (c *baseClient) process(ctx context.Context, cmd Cmder) error { - // Start measuring total operation duration (includes all retries) - // Only call time.Now() if operation duration callback is set to avoid overhead - var operationStart time.Time - opDurationCallback := otel.GetOperationDurationCallback() - if opDurationCallback != nil { - operationStart = time.Now() +// cscTrackingRequested reports whether initConn must issue CLIENT TRACKING ON. +// cscActive is allocated only after attachment succeeds and is shared with +// derived clients: a conn initialized by Conn/Tx may later return to the +// parent's pool, but a configured cache whose attachment failed must not turn +// tracking on. +func (c *baseClient) cscTrackingRequested() bool { + if c.opt.Protocol != 3 || c.cscActive == nil || !c.cscActive.Load() { + return false } + return c.opt.DB == 0 +} + +func (c *baseClient) process(ctx context.Context, cmd Cmder) error { + opDurationCallback := otel.GetOperationDurationCallback() + if opDurationCallback == nil { + return c.processCommand(ctx, cmd, nil) + } + + start := time.Now() + var state processState + err := c.processCommand(ctx, cmd, &state) + opDurationCallback(ctx, time.Since(start), cmd, state.attempts, err, state.lastConn, c.opt.DB) + return err +} + +type processState struct { + attempts int + lastConn *pool.Conn +} + +func (c *baseClient) processCommand(ctx context.Context, cmd Cmder, state *processState) error { + // Reject commands that would make one pooled connection diverge from CSC's + // tracking or database assumptions. Pipelines mirror this guard below. + if err := c.cscCommandError(cmd); err != nil { + return err + } + if c.csc != nil && isCacheable(cmd) { + return c.processCached(ctx, cmd, state) + } + return c.processWithRetry(ctx, cmd, nil, state) +} + +// processWithRetry runs cmd through the retry loop. capture (optional) is +// filled by the successful attempt's reply read for the CSC fetch path (see +// cscFetchCapture). +func (c *baseClient) processWithRetry( + ctx context.Context, cmd Cmder, capture *cscFetchCapture, state *processState, +) error { var lastConn *pool.Conn var lastErr error totalAttempts := 0 - for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ { + maxRetries := c.opt.MaxRetries + himportRetried := false + for attempt := 0; attempt <= maxRetries; attempt++ { totalAttempts++ attempt := attempt - retry, cn, err := c._process(ctx, cmd, attempt) + retry, cn, err := c._process(ctx, cmd, attempt, capture) if cn != nil { lastConn = cn } + if state != nil { + state.attempts = totalAttempts + state.lastConn = lastConn + } + // A "no such fieldset" reply for a registered fieldset means the + // connection lost its server session state (e.g. RESET, concurrent + // discard). The stale prepared flag was invalidated inside _process + // while the connection was still held; grant a single extra attempt + // so the retry re-prepares lazily on whichever connection it lands. + if err != nil && !retry && !himportRetried && !cmd.NoRetry() && + c.himportShouldRetrySet(cmd, err) { + himportRetried = true + if attempt == maxRetries { + maxRetries++ + } + lastErr = err + continue + } // Don't retry if command explicitly disables retries (e.g., RawWriteToCmd // which writes directly to an io.Writer and cannot undo partial writes) if err == nil || !retry || cmd.NoRetry() { - // Record total operation duration - if opDurationCallback != nil { - operationDuration := time.Since(operationStart) - opDurationCallback(ctx, operationDuration, cmd, totalAttempts, err, lastConn, c.opt.DB) - } - if err != nil { if errorCallback := pool.GetMetricErrorCallback(); errorCallback != nil { errorType, statusCode, isInternal := classifyCommandError(err) @@ -874,12 +1252,6 @@ func (c *baseClient) process(ctx context.Context, cmd Cmder) error { lastErr = err } - // Record failed operation after all retries - if opDurationCallback != nil { - operationDuration := time.Since(operationStart) - opDurationCallback(ctx, operationDuration, cmd, totalAttempts, lastErr, lastConn, c.opt.DB) - } - // Record error metric for exhausted retries if errorCallback := pool.GetMetricErrorCallback(); errorCallback != nil { errorType, statusCode, isInternal := classifyCommandError(lastErr) @@ -949,13 +1321,7 @@ func classifyCommandError(err error) (errorType, statusCode string, isInternal b return "UNKNOWN", "UNKNOWN", true } -func (c *baseClient) assertUnstableCommand(cmd Cmder) (bool, error) { - // All search commands (FTSearchCmd, AggregateCmd, FTInfoCmd, FTSpellCheckCmd, FTSynDumpCmd) - // now have stable RESP3 parsing. No commands require the UnstableResp3 flag anymore. - return false, nil -} - -func (c *baseClient) _process(ctx context.Context, cmd Cmder, attempt int) (bool, *pool.Conn, error) { +func (c *baseClient) _process(ctx context.Context, cmd Cmder, attempt int, capture *cscFetchCapture) (bool, *pool.Conn, error) { if attempt > 0 { if err := internal.Sleep(ctx, c.retryBackoff(attempt)); err != nil { return false, nil, err @@ -963,7 +1329,7 @@ func (c *baseClient) _process(ctx context.Context, cmd Cmder, attempt int) (bool } var usedConn *pool.Conn - retryTimeout := uint32(0) + var retryTimeout atomic.Uint32 if err := c.withConn(ctx, func(ctx context.Context, cn *pool.Conn) error { usedConn = cn // Process any pending push notifications before executing the command @@ -971,41 +1337,110 @@ func (c *baseClient) _process(ctx context.Context, cmd Cmder, attempt int) (bool internal.Logger.Printf(ctx, "push: error processing pending notifications before command: %v", err) } + // HIMPORT bookkeeping: pending discards for this session and the + // PREPARE for an HIMPORT SET's registered fieldset are written in + // the same round trip, right before the command. + var injected []Cmder + if _, ok := cmd.(himportCmder); ok { + injected = c.himportInjectedCmds(ctx, cn, []Cmder{cmd}) + } + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } return writeCmd(wr, cmd) }); err != nil { - atomic.StoreUint32(&retryTimeout, 1) + retryTimeout.Store(1) return err } readReplyFunc := cmd.readReply - // Apply unstable RESP3 search module. - if c.opt.Protocol != 2 { - useRawReply, err := c.assertUnstableCommand(cmd) - if err != nil { - return err - } - if useRawReply { - readReplyFunc = cmd.readRawReply + // When the caller requested raw-reply capture (client-side cache), + // read the reply as raw RESP bytes and re-parse them through the + // command's normal reply handler. This reuses proto.Reader rather + // than duplicating parsing logic in a bespoke cache serializer. + if capture != nil { + origRead := readReplyFunc + readReplyFunc = func(rd *proto.Reader) error { + raw, err := rd.ReadRawReply() + if err != nil { + return err + } + capture.raw = raw + return origRead(proto.NewReaderSize(bytes.NewReader(raw), len(raw)+1)) } } - if err := cn.WithReader(c.context(ctx), c.cmdTimeout(cmd), func(rd *proto.Reader) error { + readErr := cn.WithReader(c.context(ctx), c.cmdTimeout(cmd), func(rd *proto.Reader) error { // To be sure there are no buffered push notifications, we process them before reading the reply if err := c.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) } - return readReplyFunc(rd) - }); err != nil { - if cmd.readTimeout() == nil { - atomic.StoreUint32(&retryTimeout, 1) - } else { - atomic.StoreUint32(&retryTimeout, 0) + if len(injected) > 0 { + if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + return err + } + // A push notification can arrive between the injected + // replies and the command reply; drain again so the + // reply read below does not consume it as the command's. + if err := c.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil { + internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err) + } + } + err := readReplyFunc(rd) + // Assert the command type before touching the error: the + // errors.As chain inside himportNoSuchFieldset allocates, and + // this is the per-command hot path. + if set, ok := cmd.(*HImportSetCmd); ok && himportNoSuchFieldset(err) { + // A failed injected PREPARE is the root cause of the + // command's "no such fieldset" reply (drained above). + for _, ic := range injected { + if prep, ok := ic.(*HImportPrepareCmd); ok && + prep.fieldsetName == set.fieldsetName && prep.Err() != nil { + err = prep.Err() + break + } + } + // The session lost a registered fieldset the flags claim is + // prepared — and the same event (failover, cross-region + // switch, reset storm) may have wiped other sessions whose + // flags also still look current. Bump the fieldset version + // so every connection re-prepares before its next use, + // wherever the retry granted by process() lands. + if himportNoSuchFieldset(err) { + if fs, registered := c.himport.lookup(set.fieldsetName); registered { + c.himport.refreshVersion(set.fieldsetName, fs.version) + } + } } return err + }) + // redis.Nil is a complete, valid negative reply. For a CSC fetch, retain + // its connection attribution before returning Nil to the caller so the + // raw reply can be cached and invalidated like any other read result. + if readErr != nil && (capture == nil || readErr != Nil) { + if cmd.readTimeout() == nil { + retryTimeout.Store(1) + } else { + retryTimeout.Store(0) + } + return readErr + } + if capture != nil { + // Attribute while the conn is still held: once it is released, a + // queued handoff may swap the socket and bump the generation, and + // this capture is what fulfillCached compares against. + capture.connID = cn.GetID() + capture.initGen = c.cscConnInitGen(capture.connID) } - return nil + if hc, ok := cmd.(himportCmder); ok { + c.himportAfterCmd(cn, hc) + } + return readErr }); err != nil { - retry := shouldRetry(err, atomic.LoadUint32(&retryTimeout) == 1) + retry := shouldRetry(err, retryTimeout.Load() == 1) return retry, usedConn, err } @@ -1064,6 +1499,13 @@ func (c *baseClient) enableMaintNotificationsUpgrades() error { // Initialize pool hook (safe to call without lock since manager is now set) manager.InitPoolHook(c.dialHook) + // If a dedicated pipeline connection pool is in use, attach an independent + // maintnotifications hook to it as well. Otherwise autopipelined/pipelined + // commands run on pipeline-pool connections that never receive MOVING/ + // MIGRATING handoff handling. + if c.pipelinePool != nil { + manager.InitPoolHookForPool(c.pipelinePool, c.dialHook) + } return nil } @@ -1088,8 +1530,30 @@ func (c *baseClient) disableMaintNotificationsUpgrades() error { // It is rare to Close a Client, as the Client is meant to be // long-lived and shared between many goroutines. func (c *baseClient) Close() error { + // The pools this baseClient owns are shared with every WithTimeout/ + // WithReadTimeout clone. Once ANY sharer closes them, no wrapper may + // build a fresh autopipeliner against them — its flushers would run + // against closed pools forever. The atomic is checked by the + // AutoPipeline getters of every wrapper sharing this base. + if c.apClosed != nil { + c.apClosed.Store(true) + } + if h := c.cscDrainHandle; h != nil { + h.closeOnce.Do(func() { + h.closeErr = c.closeResources() + }) + return h.closeErr + } + return c.closeResources() +} + +func (c *baseClient) closeResources() error { var firstErr error + // CSC teardown (no-op when CSC is not active): stop the background + // invalidation drainer before the pool it walks is torn down. + c.stopBackgroundDrainer() + // Close maintnotifications manager first if err := c.disableMaintNotificationsUpgrades(); err != nil { firstErr = err @@ -1100,13 +1564,18 @@ func (c *baseClient) Close() error { } // Unregister pools from OTel before closing them - otel.UnregisterPools(c.connPool, c.pubSubPool) + otel.UnregisterPools(c.connPool, c.pubSubPool, c.pipelinePool) if c.connPool != nil { if err := c.connPool.Close(); err != nil && firstErr == nil { firstErr = err } } + if c.pipelinePool != nil { + if err := c.pipelinePool.Close(); err != nil && firstErr == nil { + firstErr = err + } + } if c.pubSubPool != nil { if err := c.pubSubPool.Close(); err != nil && firstErr == nil { firstErr = err @@ -1138,6 +1607,14 @@ type pipelineProcessor func(context.Context, *pool.Conn, []Cmder) (bool, error) func (c *baseClient) generalProcessPipeline( ctx context.Context, cmds []Cmder, p pipelineProcessor, operationName string, ) error { + // Pipeline commands never pass through process, so apply the same CSC state + // guard here. initConn's internal client is exempt. + for _, cmd := range cmds { + if err := c.cscCommandError(cmd); err != nil { + setCmdsErr(cmds, err) + return err + } + } // Only call time.Now() if pipeline operation duration callback is set to avoid overhead var operationStart time.Time pipelineOpDurationCallback := otel.GetPipelineOperationDurationCallback() @@ -1163,7 +1640,9 @@ func (c *baseClient) generalProcessPipeline( // Enable retries by default to retry dial errors returned by withConn. canRetry := true - lastErr = c.withConn(ctx, func(ctx context.Context, cn *pool.Conn) error { + // Route pipelines through the dedicated pipeline pool when configured; + // withPipelineConn falls back to the regular pool when it is not. + lastErr = c.withPipelineConn(ctx, func(ctx context.Context, cn *pool.Conn) error { lastConn = cn // Process any pending push notifications before executing the pipeline if err := c.processPushNotifications(ctx, cn); err != nil { @@ -1196,6 +1675,17 @@ func (c *baseClient) generalProcessPipeline( } } + // Retries exhausted on a retryable error: the loop fell through without the + // early-exit branch running, so the commands were never populated with the + // failure. Mirror that branch here so callers that observe results only + // per-command — notably AutoPipeline, which discards this function's returned + // error — see the error instead of a nil error and a zero value. Guard on + // !isRedisError so a per-command redis error (e.g. LOADING) keeps its own + // reply rather than being overwritten. + if !isRedisError(lastErr) { + setCmdsErr(cmds, lastErr) + } + if pipelineOpDurationCallback != nil { operationDuration := time.Since(operationStart) pipelineOpDurationCallback(ctx, operationDuration, operationName, len(cmds), totalAttempts, lastErr, lastConn, c.opt.DB) @@ -1217,21 +1707,69 @@ func (c *baseClient) pipelineProcessCmds( internal.Logger.Printf(ctx, "push: error processing pending notifications before writing pipeline: %v", err) } + // HIMPORT bookkeeping: pending discards for this session and PREPAREs + // for registered fieldsets the batch references get written ahead of + // the batch. + injected := c.himportInjectedCmds(ctx, cn, cmds) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } return writeCmds(wr, cmds) }); err != nil { setCmdsErr(cmds, err) return true, err } + var readErr error if err := cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { + if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + // Transport error with every batch reply unreadXX: stamp the + // batch like a write failure. The outer retry loop stamps only + // on its exit branch, not when attempts run out, so without + // this a batch that keeps dying here would surface an Exec + // error while every command still reports Err() == nil. + setCmdsErr(cmds, err) + return err + } // read all replies - return c.pipelineReadCmds(ctx, cn, rd, cmds) + readErr = c.pipelineReadCmds(ctx, cn, rd, cmds) + if readErr != nil && !isRedisError(readErr) { + return readErr + } + c.himportAfterBatch(cn, injected, cmds) + return nil }); err != nil { return true, err } - return false, nil + // Registered fieldsets whose SETs came back "no such fieldset" (the + // session was lost between prepare and use) are re-prepared and those + // SETs re-issued once on the same connection; the error must not + // surface for managed fieldsets. + // + // A transport failure here must neither retry nor fail the batch: the + // first round trip was fully consumed and its results delivered, so + // re-executing would double-apply non-idempotent commands and failing + // would stamp a spurious error onto commands that succeeded. The + // re-issue errors stay on the retried SETs; the connection, which may + // hold unread replies, is marked for removal when released. + if err := c.himportRetryFailedSets(ctx, cn, cmds); err != nil { + internal.Logger.Printf(ctx, "himport: pipeline set re-issue failed: %v", err) + cn.MarkCloseOnPut("himport: transport error during set re-issue") + } + + // Preserve retryable first-command errors (e.g. LOADING) for the outer + // loop; the re-issue above may have cleared it. rawErr: this runs on the + // execution path; never await here (an async autopipeline command's ready + // channel is closed by this very batch — Err() would self-deadlock). + if readErr != nil { + readErr = cmds[0].rawErr() + } + return readErr != nil, readErr } func (c *baseClient) pipelineReadCmds(ctx context.Context, cn *pool.Conn, rd *proto.Reader, cmds []Cmder) error { @@ -1248,7 +1786,9 @@ func (c *baseClient) pipelineReadCmds(ctx context.Context, cn *pool.Conn, rd *pr } } // Retry errors like "LOADING redis is loading the dataset in memory". - return cmds[0].Err() + // rawErr: this runs on the execution path; never await here (an async + // autopipeline command's ready channel is closed by this very batch). + return cmds[0].rawErr() } func (c *baseClient) txPipelineProcessCmds( @@ -1259,7 +1799,17 @@ func (c *baseClient) txPipelineProcessCmds( internal.Logger.Printf(ctx, "push: error processing pending notifications before transaction: %v", err) } + // HIMPORT bookkeeping: pending discards for this session and PREPAREs + // for registered fieldsets the transaction references get written ahead + // of MULTI; the session state is visible inside the transaction. + injected := c.himportInjectedCmds(ctx, cn, cmds) + if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error { + for _, ic := range injected { + if err := writeCmd(wr, ic); err != nil { + return err + } + } return writeCmds(wr, cmds) }); err != nil { setCmdsErr(cmds, err) @@ -1267,6 +1817,13 @@ func (c *baseClient) txPipelineProcessCmds( } if err := cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error { + if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil { + // Transport error with every transaction reply unread: stamp + // the batch like a write failure (see pipelineProcessCmds). + setCmdsErr(cmds, err) + return err + } + statusCmd := cmds[0].(*StatusCmd) // Trim multi and exec. trimmedCmds := cmds[1 : len(cmds)-1] @@ -1277,7 +1834,11 @@ func (c *baseClient) txPipelineProcessCmds( } // Read replies. - return c.pipelineReadCmds(ctx, cn, rd, trimmedCmds) + err := c.pipelineReadCmds(ctx, cn, rd, trimmedCmds) + if err == nil || isRedisError(err) { + c.himportAfterBatch(cn, injected, trimmedCmds) + } + return err }); err != nil { return false, err } @@ -1341,6 +1902,16 @@ func (c *baseClient) txPipelineReadQueued(ctx context.Context, cn *pool.Conn, rd type Client struct { *baseClient cmdable + + // cscLifecycleOwner keeps the canonical Client wrapper (the one whose GC + // cleanup owns the drainer) reachable while a WithTimeout clone can still + // serve from its cache. Nil on the canonical wrapper and on non-CSC clones. + cscLifecycleOwner *Client + + autopipelinerMu *sync.Mutex // guards the autopipeliner fields against concurrent first-call creation + autopipeliner *AutoPipeliner // blocking face (Client.AutoPipeline) + asyncAutopipeliner *AutoPipeliner // deferred face (Client.AsyncAutoPipeline) + autopipelinerClosed bool // set by Close: refuse to resurrect a pipeliner on a closed client } // NewClient returns a client to the Redis Server specified by Options. @@ -1357,8 +1928,10 @@ func NewClient(opt *Options) *Client { c := Client{ baseClient: &baseClient{ - opt: opt, - onClose: &onCloseHooks{}, + apClosed: &atomic.Bool{}, + opt: opt, + onClose: &onCloseHooks{}, + himport: newHImportRegistry(), }, } c.init() @@ -1385,9 +1958,61 @@ func NewClient(opt *Options) *Client { panic(fmt.Errorf("redis: failed to create pubsub pool: %w", err)) } + // Optionally create a separate connection pool for pipelining, with its own + // (typically larger) buffers, so pipelines can use big buffers without + // bloating the main pool. Enabled when either pipeline buffer size is set. + if opt.PipelineReadBufferSize > 0 || opt.PipelineWriteBufferSize > 0 { + pipelineOpt := opt.clone() + if opt.PipelineReadBufferSize > 0 { + pipelineOpt.ReadBufferSize = opt.PipelineReadBufferSize + // Same clamp Options.init applies to the main pool: RESP3 push + // parsing needs a minimum read buffer, and a tiny pipeline reader + // would break push-notification handling on pipeline conns. + if pipelineOpt.Protocol == 3 && pipelineOpt.ReadBufferSize < proto.MinRESP3ReadBufferSize { + pipelineOpt.ReadBufferSize = proto.MinRESP3ReadBufferSize + } + } + if opt.PipelineWriteBufferSize > 0 { + pipelineOpt.WriteBufferSize = opt.PipelineWriteBufferSize + } + if opt.PipelinePoolSize > 0 { + pipelineOpt.PoolSize = opt.PipelinePoolSize + } else { + pipelineOpt.PoolSize = 10 // default smaller pool for pipelining + } + pipelinePoolName := opt.Addr + "_" + uniqueID + "_pipeline" + c.pipelinePoolName = pipelinePoolName + c.pipelinePool, err = newConnPool(pipelineOpt, c.dialHook, pipelinePoolName) + if err != nil { + panic(fmt.Errorf("redis: failed to create pipeline connection pool: %w", err)) + } + } + if opt.StreamingCredentialsProvider != nil { c.streamingCredentialsManager = streaming.NewManager(c.connPool, c.opt.PoolTimeout) c.connPool.AddPoolHook(c.streamingCredentialsManager.PoolHook()) + if c.pipelinePool != nil { + c.pipelinePool.AddPoolHook(c.streamingCredentialsManager.PoolHook()) + } + } + + // CSC wiring (SharedTracking): shared cache + per-connection CLIENT TRACKING + + // background drainer. attachCSC is the strategy dispatch entry. + if opt.Protocol == 3 { + var cache Cache + if explicit := opt.ClientSideCache; explicit != nil { + cache = explicit + } else if cfg := opt.ClientSideCacheConfig; cfg != nil { + cache = NewLocalCache(*cfg) + // We constructed it, so we own it (may flush on drainer stop). + c.baseClient.cscOwnsCache = true + } + c.baseClient.attachCSC(context.Background(), cache) + + // Safety net for a client dropped without Close: the goroutines hold + // *baseClient (never *Client), so dropping *Client (returned as &c) + // triggers these cleanups, which stop them. See cscRegisterCleanups. + cscRegisterCleanups(&c) } // Initialize maintnotifications first if enabled and protocol is RESP3 @@ -1413,12 +2038,17 @@ func NewClient(opt *Options) *Client { // Register pools with OTel recorder if it supports pool registration // This allows async gauge metrics to pull stats from pools periodically - otel.RegisterPools(c.connPool, c.pubSubPool, opt.Addr) + otel.RegisterPools(c.connPool, c.pubSubPool, c.pipelinePool, opt.Addr) return &c } func (c *Client) init() { + // Fresh per-Client guard and no inherited autopipeliner: a WithTimeout clone + // (clone := *c) must not share the parent's mutex or AutoPipeliner instance. + c.autopipelinerMu = &sync.Mutex{} + c.autopipeliner = nil + c.asyncAutopipeliner = nil c.cmdable = c.Process c.initHooks(hooks{ dial: c.baseClient.dial, @@ -1428,15 +2058,89 @@ func (c *Client) init() { }) } +// WithTimeout returns a clone sharing the parent's connection pools with the +// given read/write timeout. The clone caches its own autopipeliners: an +// AutoPipeline()/AsyncAutoPipeline() created on the clone is NOT stopped by +// the parent's Close — call Close on the clone's autopipeliner explicitly. func (c *Client) WithTimeout(timeout time.Duration) *Client { + // Snapshot under the guard: AutoPipeline()/Close() mutate the + // autopipeliner fields concurrently, so a bare struct copy of them is a + // data race (init below discards the copied values either way). + c.autopipelinerMu.Lock() clone := *c + c.autopipelinerMu.Unlock() + if c.cscLifecycleOwner != nil { + clone.cscLifecycleOwner = c.cscLifecycleOwner + } else if c.baseClient.cscDrainHandle != nil { + clone.cscLifecycleOwner = c + } clone.baseClient = c.baseClient.withTimeout(timeout) clone.init() return &clone } +// Close closes the client, stopping both cached autopipeliners (the blocking +// AutoPipeline instance and the async AsyncAutoPipeline instance, if created) +// before releasing the underlying resources, so their background flusher +// goroutines don't outlive the client. AutoPipeliner.Close is idempotent and +// safe to call here even if autopipelining was never used. +// A WithTimeout clone delegates CSC teardown to the canonical wrapper that +// owns the background drainer. +func (c *Client) Close() error { + c.autopipelinerMu.Lock() + ap, async := c.autopipeliner, c.asyncAutopipeliner + c.autopipeliner, c.asyncAutopipeliner = nil, nil + // A later AutoPipeline()/AsyncAutoPipeline() call must not build a fresh + // pipeliner against the closed pools: nothing would ever close it and its + // flusher goroutines would leak. The getters check this flag. + c.autopipelinerClosed = true + c.autopipelinerMu.Unlock() + var firstErr error + for _, p := range []*AutoPipeliner{ap, async} { + if p != nil { + if err := p.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + } + if c.cscLifecycleOwner != nil { + // Delegate through the OWNER's *Client.Close, not its baseClient: + // the owner may hold cached autopipeliners of its own whose flusher + // goroutines must stop with the shared pools, and its + // autopipelinerClosed flag must flip so later owner getters cannot + // resurrect a pipeliner against closed pools. Client.Close is + // idempotent through baseClient.Close, so an owner also closed + // directly is fine. + if err := c.cscLifecycleOwner.Close(); err != nil && firstErr == nil { + firstErr = err + } + return firstErr + } + if err := c.baseClient.Close(); err != nil && firstErr == nil { + firstErr = err + } + return firstErr +} + func (c *Client) Conn() *Conn { - return newConn(c.opt, pool.NewStickyConnPool(c.connPool), &c.hooksMixin) + // Share the HIMPORT fieldset registry: the sticky pool borrows + // connections from this client's pool, so fieldsets prepared on them + // stay valid after the connections are returned. + conn := newConn(c.opt, c.baseClient.newStickyConnPool(), &c.hooksMixin, c.himport) + // A sticky client does not serve cache hits, but a new pool connection first + // initialized through it may later be reused by the parent. Share the + // successful-attachment signal so that connection is tracked exactly when + // the parent's CSC is active. + conn.baseClient.cscActive = c.baseClient.cscActive + // No-op today: the strategy needs an idle-conn drainer and a StickyConnPool + // has none, so CSC isn't active on a Conn() (its reads hit the server). Kept + // so a future sticky-pool-capable strategy attaches here. + conn.baseClient.attachCSC(context.Background(), c.csc) + // Carry the parent's shared eviction hook so that if this derived client + // initializes a pool conn, the close hook it installs still evicts from the + // parent cache (its own csc is nil). + conn.baseClient.cscPoolHook = c.baseClient.cscPoolHook + return conn } func (c *Client) Process(ctx context.Context, cmd Cmder) error { @@ -1505,7 +2209,10 @@ type PoolStats pool.Stats // PoolStats returns connection pool stats. func (c *Client) PoolStats() *PoolStats { stats := c.connPool.Stats() - stats.PubSubStats = *(c.pubSubPool.Stats()) + stats.PubSubStats = *c.pubSubPool.Stats() + if c.pipelinePool != nil { + stats.PipelineStats = c.pipelinePool.Stats() + } return (*PoolStats)(stats) } @@ -1521,6 +2228,79 @@ func (c *Client) Pipeline() Pipeliner { return &pipe } +// AutoPipeline returns the blocking autopipeliner for this client: a drop-in +// replacement for the normal command surface where each command call (ap.Set, +// ap.Get, ...) blocks until executed, exactly like a plain client — but the +// engine batches concurrent callers' commands into pipelines, so throughput is +// far higher (measured locally over loopback: ~1M+ SET/sec vs ~100k; indicative, not a guarantee). Commands keep per-goroutine order. +// +// By default, Options.AutoPipelineOptions is used if set, +// otherwise DefaultBlockingAutoPipelineOptions (a single ordered batch stream, +// which maximizes throughput and minimizes latency for the blocking face — see +// its doc). The instance is cached and shared; the first +// call's config wins and later calls return the same instance until it is closed. +// It must be closed (or close the client) to release its goroutines. +// +// It returns an error if the supplied config is invalid (e.g. MaxConcurrentBatches>1 +// without Unordered, or a negative size); on error no instance is cached. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *Client) AutoPipeline() (*AutoPipeliner, error) { + return c.AutoPipelineWithOptions(nil) +} + +// AutoPipelineWithOptions is AutoPipeline with explicit options instead of +// Options.AutoPipelineOptions / the default. The instance is cached and shared; +// the first call's config wins. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *Client) AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return getOrCreateAutoPipeliner(c.autopipelinerMu, &c.autopipeliner, &c.autopipelinerClosed, c.baseClient.apClosed, config, + func() *AutoPipelineOptions { + if c.opt.AutoPipelineOptions != nil { + return c.opt.AutoPipelineOptions + } + return DefaultBlockingAutoPipelineOptions() + }, + func(cfg *AutoPipelineOptions) (*AutoPipeliner, error) { return newAutoPipeliner(c, cfg, true) }) +} + +// AsyncAutoPipeline returns the deferred (async) autopipeliner: command calls +// return immediately and the result accessors (Val/Result/Err) block until the +// command has executed. Submit a window of commands, then read their results, to +// keep each pipeline deep and reach the highest throughput (measured locally over loopback: ~2-3M SET/sec; indicative). +// +// By default, Options.AutoPipelineOptions is used if set, +// otherwise DefaultAutoPipelineOptions (ordered, MaxConcurrentBatches: 1) — a +// single goroutine's deferred commands execute in submit order. Use AsyncAutoPipelineWithOptions +// to override (and, for parallel batches, set Unordered). The instance is +// cached and shared; the first call's config wins. Close it (or the client) to +// release its goroutines. +// +// It returns an error if the supplied config is invalid (e.g. MaxConcurrentBatches>1 +// without Unordered, or a negative size); on error no instance is cached. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *Client) AsyncAutoPipeline() (*AutoPipeliner, error) { + return c.AsyncAutoPipelineWithOptions(nil) +} + +// AsyncAutoPipelineWithOptions is AsyncAutoPipeline with an explicit config +// instead of Options.AutoPipelineOptions / the default. The instance is cached +// and shared; the first call's config wins. +// +// EXPERIMENTAL: this API is subject to change, use with caution. +func (c *Client) AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return getOrCreateAutoPipeliner(c.autopipelinerMu, &c.asyncAutopipeliner, &c.autopipelinerClosed, c.baseClient.apClosed, config, + func() *AutoPipelineOptions { + if c.opt.AutoPipelineOptions != nil { + return c.opt.AutoPipelineOptions + } + return DefaultAutoPipelineOptions() + }, + func(cfg *AutoPipelineOptions) (*AutoPipeliner, error) { return newAutoPipeliner(c, cfg, false) }) +} + func (c *Client) TxPipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { return c.TxPipeline().Pipelined(ctx, fn) } @@ -1539,7 +2319,7 @@ func (c *Client) TxPipeline() Pipeliner { func (c *Client) pubSub() *PubSub { pubsub := &PubSub{ - opt: c.opt, + opt: c.cloneOpt(), newConn: func(ctx context.Context, addr string, channels []string) (*pool.Conn, error) { cn, err := c.pubSubPool.NewConn(ctx, c.opt.Network, addr, channels) if err != nil { @@ -1634,14 +2414,20 @@ type Conn struct { } // newConn is a helper func to create a new Conn instance. -// the Conn instance is not thread-safe and should not be shared between goroutines. -// the parentHooks will be cloned, no need to clone before passing it. -func newConn(opt *Options, connPool pool.Pooler, parentHooks *hooksMixin) *Conn { +// The Conn instance is not thread-safe and should not be shared between goroutines. +// The parentHooks will be cloned, no need to clone before passing it. +// himport is the HIMPORT fieldset registry the Conn participates in — pass +// the owning client's registry (a private one would restart versions at 1 +// and collide with the client's version space on the shared pooled +// connections); nil disables HIMPORT tracking. +func newConn(opt *Options, connPool pool.Pooler, parentHooks *hooksMixin, himport *himportRegistry) *Conn { c := Conn{ baseClient: baseClient{ + apClosed: &atomic.Bool{}, opt: opt, connPool: connPool, onClose: &onCloseHooks{}, + himport: himport, }, } @@ -1739,25 +2525,116 @@ func (c *baseClient) processPushNotifications(ctx context.Context, cn *pool.Conn } } - // Check if there is any data to read before processing - // This is an optimization on UNIX systems where MaybeHasData is a syscall - // On Windows, MaybeHasData always returns true, so this check is a no-op + return c.peekAndProcessPushNotifications(ctx, cn) +} + +// peekAndProcessPushNotifications peeks the socket and processes any pending +// push notifications on cn unconditionally, bypassing the recent-health-check +// shortcut in processPushNotifications. Required on paths that do not follow +// up with a reply read on the same connection (e.g. the CSC cache-hit drain), +// where the shortcut would otherwise suppress invalidations buffered since the +// last health check. +func (c *baseClient) peekAndProcessPushNotifications(ctx context.Context, cn *pool.Conn) error { + if c.opt.Protocol != 3 || c.pushProcessor == nil { + return nil + } + if !cn.MaybeHasData() { return nil } - // Use WithReader to access the reader and process push notifications - // This is critical for maintnotifications to work properly - // NOTE: almost no timeouts are set for this read, so it should not block - // longer than necessary, 10us should be plenty of time to read if there are any push notifications - // on the socket. - return cn.WithReader(ctx, 10*time.Microsecond, func(rd *proto.Reader) error { - // Create handler context with client, connection pool, and connection information + // Short read timeout: MaybeHasData confirmed kernel-buffered bytes, so + // the first read returns immediately — the deadline only needs to cover + // scheduler pauses, not network waits. 10us was routinely lost to + // scheduling on loaded machines: the peek then timed out with nothing + // consumed, the processor treated that as "no pending data", and a + // connection with buffered push bytes was returned to the pool instead + // of being drained (or removed, when the frame turns out partial). + return cn.WithReader(ctx, time.Millisecond, func(rd *proto.Reader) error { handlerCtx := c.pushNotificationHandlerContext(cn) return c.pushProcessor.ProcessPendingNotifications(ctx, handlerCtx, rd) }) } +// cscFallbackProbeInterval bounds how often an idle connection without a +// portable readiness mechanism is subjected to a timed read. Post-command +// probes remain immediate; this is only the eventual invalidation fallback. +const cscFallbackProbeInterval = 100 * time.Millisecond + +// drainPushNotifications drains push frames buffered on a connection the CSC +// drainer has claimed, under a HARD read deadline. processorSucceeded reports a +// successful processor invocation; it resets custom-processor damping even when +// the frame was hidden inside a transport wrapper. A non-nil error is +// connection-fatal (the drainer removes the conn), including a read timeout +// after reply consumption starts: the reader may be desynchronized. A custom +// processor's error is also fatal because its contract cannot prove no bytes +// were consumed. +func (c *baseClient) drainPushNotifications(cn *pool.Conn) (processorSucceeded bool, err error) { + if c.opt.Protocol != 3 || c.pushProcessor == nil { + return false, nil + } + // Skip only when nothing is buffered (reader) AND nothing on the socket: + // MaybeHasData peeks only the socket, but an invalidate can sit in cn.rd. + readPending := cn.TakeCscReadPending() + periodicReadPending := cn.TakeCscPeriodicReadPending(cscFallbackProbeInterval) + socketData, socketErr := cn.CheckForData() + if socketErr != nil { + return false, socketErr + } + hasData := cn.HasBufferedData() || socketData + if !readPending && !periodicReadPending && !hasData { + return false, nil + } + if !hasData { + // TLS and opaque wrappers can hide bytes from the socket readiness + // check. Probe one byte without consuming it under a tiny deadline; + // only a confirmed byte gets the longer fragmented-frame budget below. + err := cn.WithReaderHardDeadline(cscDrainProbeReadCap, func(rd *proto.Reader) error { + _, err := rd.Peek(1) + return err + }) + if err != nil { + if isTimeout, hasTimeoutFlag := isTimeoutError(err); isTimeout && hasTimeoutFlag { + return false, nil + } + return false, err + } + } + + handlerCtx := c.pushNotificationHandlerContext(cn) + handlerCtx.Client = cscHandlerClient{baseClient: c} + err = cn.WithReaderHardDeadline(cscDrainHardReadCap, func(rd *proto.Reader) error { + if processor, ok := c.pushProcessor.(*push.Processor); ok { + return processor.ProcessPendingNotificationsBuffered( + context.Background(), handlerCtx, rd) + } + return c.pushProcessor.ProcessPendingNotifications(context.Background(), handlerCtx, rd) + }) + if err != nil { + // The built-in processor surfaces mid-frame ReadReply errors (a benign + // boundary peek timeout returns nil). allowTimeout=false: such an error + // means bytes were consumed mid-frame, leaving the conn desynced — + // re-pooling would corrupt the next command's reply, so remove it. + if _, builtin := c.pushProcessor.(*push.Processor); builtin { + if isBadConn(err, false, c.opt.Addr) { + return true, err // fatal read/protocol/connection error — remove the conn + } + return true, nil + } + // A CUSTOM processor's error contract is unknown: it may have consumed + // part of a frame before failing, and a mid-frame reader silently + // corrupts the next command's reply. The conn is idle and held solely + // by the drainer, so the safe default — removal — costs one reconnect; + // persistent failures are damped by the drainer (cscDrainCustomErrCap). + internal.Logger.Printf(context.Background(), "csc: drain: custom push processor error (removing conn): %v", err) + return true, err + } + // The processor ran successfully. This is stronger evidence than a clean + // connection on which it was never invoked, and prevents successful TLS- + // buffered drains from being counted as if failures were consecutive. + return true, nil +} + // processPendingPushNotificationWithReader processes all pending push notifications on a connection // This method should be called by the client in WithReader before reading the reply func (c *baseClient) processPendingPushNotificationWithReader(ctx context.Context, cn *pool.Conn, rd *proto.Reader) error { diff --git a/backend/vendor/github.com/redis/go-redis/v9/ring.go b/backend/vendor/github.com/redis/go-redis/v9/ring.go index b60d3eab..a22e1667 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/ring.go +++ b/backend/vendor/github.com/redis/go-redis/v9/ring.go @@ -48,6 +48,10 @@ type RingOptions struct { // NewClient creates a shard client with provided options. NewClient func(opt *Options) *Client + // himport is the ring-wide HIMPORT fieldset registry, set by NewRing and + // shared with every shard client (see himport.go, himport_cluster.go). + himport *himportRegistry + // ClientName will execute the `CLIENT SETNAME ClientName` command for each conn. ClientName string @@ -143,6 +147,14 @@ type RingOptions struct { // default: 32KiB (32768 bytes) WriteBufferSize int + // PipelineReadBufferSize, PipelineWriteBufferSize and PipelinePoolSize + // configure an optional separate connection pool used for pipelining on + // each shard, with its own (typically larger) buffers. See the same-named + // fields on Options for details. The pool is created only when PipelineReadBufferSize or PipelineWriteBufferSize is set (PipelinePoolSize alone does not enable it). + PipelineReadBufferSize int + PipelineWriteBufferSize int + PipelinePoolSize int + TLSConfig *tls.Config Limiter Limiter @@ -194,13 +206,13 @@ func (opt *RingOptions) init() { case -1: opt.MinRetryBackoff = 0 case 0: - opt.MinRetryBackoff = 8 * time.Millisecond + opt.MinRetryBackoff = 10 * time.Millisecond } switch opt.MaxRetryBackoff { case -1: opt.MaxRetryBackoff = 0 case 0: - opt.MaxRetryBackoff = 512 * time.Millisecond + opt.MaxRetryBackoff = time.Second } if opt.ReadBufferSize == 0 { @@ -247,6 +259,10 @@ func (opt *RingOptions) clientOptions() *Options { ReadBufferSize: opt.ReadBufferSize, WriteBufferSize: opt.WriteBufferSize, + PipelineReadBufferSize: opt.PipelineReadBufferSize, + PipelineWriteBufferSize: opt.PipelineWriteBufferSize, + PipelinePoolSize: opt.PipelinePoolSize, + TLSConfig: opt.TLSConfig, Limiter: opt.Limiter, @@ -262,7 +278,7 @@ func (opt *RingOptions) clientOptions() *Options { type ringShard struct { Client *Client - down int32 + down atomic.Int32 addr string } @@ -270,10 +286,16 @@ func newRingShard(opt *RingOptions, addr string) *ringShard { clopt := opt.clientOptions() clopt.Addr = addr - return &ringShard{ + shard := &ringShard{ Client: opt.NewClient(clopt), addr: addr, } + // Share the ring-wide HIMPORT fieldset registry so any shard connection + // serving an HIMPORT SET can lazily replay the PREPARE. + if opt.himport != nil { + shard.Client.himport = opt.himport + } + return shard } func (shard *ringShard) String() string { @@ -288,7 +310,7 @@ func (shard *ringShard) String() string { func (shard *ringShard) IsDown() bool { const threshold = 3 - return atomic.LoadInt32(&shard.down) >= threshold + return shard.down.Load() >= threshold } func (shard *ringShard) IsUp() bool { @@ -299,7 +321,7 @@ func (shard *ringShard) IsUp() bool { func (shard *ringShard) Vote(up bool) bool { if up { changed := shard.IsDown() - atomic.StoreInt32(&shard.down, 0) + shard.down.Store(0) return changed } @@ -307,7 +329,7 @@ func (shard *ringShard) Vote(up bool) bool { return false } - atomic.AddInt32(&shard.down, 1) + shard.down.Add(1) return shard.IsDown() } @@ -369,9 +391,10 @@ func (c *ringSharding) SetAddrs(addrs map[string]string) { return } existing := c.shards + onNewNode := c.onNewNode c.mu.RUnlock() - shards, created, unused := c.newRingShards(addrs, existing) + shards, created, unused := c.newRingShards(addrs, existing, onNewNode) c.mu.Lock() if c.closed { @@ -387,7 +410,7 @@ func (c *ringSharding) SetAddrs(addrs map[string]string) { } func (c *ringSharding) newRingShards( - addrs map[string]string, existing *ringShards, + addrs map[string]string, existing *ringShards, onNewNode []func(rdb *Client), ) (shards *ringShards, created, unused map[string]*ringShard) { shards = &ringShards{m: make(map[string]*ringShard, len(addrs))} created = make(map[string]*ringShard) // indexed by addr @@ -408,7 +431,7 @@ func (c *ringSharding) newRingShards( shards.m[name] = shard created[addr] = shard - for _, fn := range c.onNewNode { + for _, fn := range onNewNode { fn(shard.Client) } } @@ -604,7 +627,16 @@ func NewRing(opt *RingOptions) *Ring { if opt == nil { panic("redis: NewRing nil options") } + // Shallow-copy the options: the ring-wide HIMPORT registry is carried + // through them to shard construction, and reusing one caller-owned + // RingOptions across several rings must not make the rings share (or + // clobber each other's) registry. + optCopy := *opt + opt = &optCopy opt.init() + // The registry must exist before the first shard is created; shards + // adopt it in newRingShard. + opt.himport = newHImportRegistry() hbCtx, hbCancel := context.WithCancel(context.Background()) @@ -714,6 +746,17 @@ func (c *Ring) SSubscribe(ctx context.Context, channels ...string) *PubSub { return shard.Client.SSubscribe(ctx, channels...) } +// Publish posts the message to the channel +func (c *Ring) Publish(ctx context.Context, channel string, message interface{}) *IntCmd { + shard, err := c.sharding.GetByKey(channel) + if err != nil { + cmd := NewIntCmd(ctx, "publish", channel, message) + cmd.SetErr(err) + return cmd + } + return shard.Client.Publish(ctx, channel, message) +} + func (c *Ring) OnNewNode(fn func(rdb *Client)) { c.sharding.OnNewNode(fn) } @@ -820,6 +863,34 @@ func (c *Ring) Pipeline() Pipeliner { return &pipe } +// ErrRingAutoPipelineUnsupported is returned by Ring's AutoPipeline / +// AsyncAutoPipeline (and their WithOptions forms); check for it with +// errors.Is. Autopipelining is not implemented for Ring; use the per-shard +// clients or a ClusterClient. Ring is part of the UniversalClient +// interface, so these methods exist to satisfy it and fail explicitly rather +// than being silently absent. +var ErrRingAutoPipelineUnsupported = errors.New("redis: AutoPipeline is not supported by Ring") + +// AutoPipeline is not supported by Ring; it returns ErrRingAutoPipelineUnsupported. +func (c *Ring) AutoPipeline() (*AutoPipeliner, error) { + return c.AutoPipelineWithOptions(nil) +} + +// AutoPipelineWithOptions is not supported by Ring; it returns ErrRingAutoPipelineUnsupported. +func (c *Ring) AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return nil, ErrRingAutoPipelineUnsupported +} + +// AsyncAutoPipeline is not supported by Ring; it returns ErrRingAutoPipelineUnsupported. +func (c *Ring) AsyncAutoPipeline() (*AutoPipeliner, error) { + return c.AsyncAutoPipelineWithOptions(nil) +} + +// AsyncAutoPipelineWithOptions is not supported by Ring; it returns ErrRingAutoPipelineUnsupported. +func (c *Ring) AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) { + return nil, ErrRingAutoPipelineUnsupported +} + func (c *Ring) TxPipelined(ctx context.Context, fn func(Pipeliner) error) ([]Cmder, error) { return c.TxPipeline().Pipelined(ctx, fn) } diff --git a/backend/vendor/github.com/redis/go-redis/v9/script.go b/backend/vendor/github.com/redis/go-redis/v9/script.go index 92d508f9..5b649f66 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/script.go +++ b/backend/vendor/github.com/redis/go-redis/v9/script.go @@ -192,7 +192,7 @@ func (s *Script) EvalShaRO(ctx context.Context, c Scripter, keys []string, args // it is retried using EVAL. func (s *Script) Run(ctx context.Context, c Scripter, keys []string, args ...interface{}) *Cmd { r := s.EvalSha(ctx, c, keys, args...) - if errors.Is(r.Err(), ErrNoScript) { + if isNoScriptErr(r.Err()) { return s.Eval(ctx, c, keys, args...) } return r @@ -202,8 +202,20 @@ func (s *Script) Run(ctx context.Context, c Scripter, keys []string, args ...int // it is retried using EVAL_RO. func (s *Script) RunRO(ctx context.Context, c Scripter, keys []string, args ...interface{}) *Cmd { r := s.EvalShaRO(ctx, c, keys, args...) - if errors.Is(r.Err(), ErrNoScript) { + if isNoScriptErr(r.Err()) { return s.EvalRO(ctx, c, keys, args...) } return r } + +// isNoScriptErr reports whether err means "this digest is not cached", whether +// it arrived already normalized to ErrNoScript or as the server's raw NOSCRIPT +// error. Both are accepted because the Eval wrappers only normalize when the +// result is readable without blocking — on the deferred autopipeline face the +// raw error reaches here untouched (see cmdable.eval). +func isNoScriptErr(err error) bool { + if err == nil { + return false + } + return errors.Is(err, ErrNoScript) || HasErrorPrefix(err, "NOSCRIPT") +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/scripting_commands.go b/backend/vendor/github.com/redis/go-redis/v9/scripting_commands.go index 3310b9d0..6845f398 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/scripting_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/scripting_commands.go @@ -60,8 +60,16 @@ func (c cmdable) eval(ctx context.Context, name, payload string, keys []string, cmd.SetFirstKeyPos(3) } _ = c(ctx, cmd) - if err := cmd.Err(); err != nil { - if HasErrorPrefix(err, "NOSCRIPT") { + // Normalize NOSCRIPT to ErrNoScript for Script.Run/RunRO's EVAL fallback, + // but only when the result is already readable: on the deferred + // autopipeline face the call above merely enqueues, and reading the outcome + // here would await execution — making the whole Eval family synchronous on + // a face whose contract is to return immediately (review finding by codex + // on #3942). When the result is still pending the normalization is skipped; + // Script.Run/RunRO also match the raw NOSCRIPT prefix, so the fallback + // keeps working on that face. + if cmd.resultReady() { + if err := cmd.rawErr(); err != nil && HasErrorPrefix(err, "NOSCRIPT") { cmd.SetErr(ErrNoScript) } } diff --git a/backend/vendor/github.com/redis/go-redis/v9/search_builders.go b/backend/vendor/github.com/redis/go-redis/v9/search_builders.go index a6c6718c..094441c7 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/search_builders.go +++ b/backend/vendor/github.com/redis/go-redis/v9/search_builders.go @@ -313,6 +313,31 @@ func (b *AggregateBuilder) ReduceAs(fn SearchAggregator, alias string, args ...i return b } +// Collect adds a REDUCE COLLECT clause to the last step, which must be a +// GROUPBY. The COLLECT options (FIELDS/DISTINCT/SORTBY/LIMIT/AS) are rendered +// and the argument count is computed automatically; field and sort names are +// normalized to a single "@" prefix. Set FTAggregateCollect.As to alias the +// output column. +// +// If the last step is not a GROUPBY, or the options are invalid (no FIELDS +// selector), Run returns the recorded error without issuing the command. +// COLLECT requires Redis 8.8+ with unstable features enabled. +func (b *AggregateBuilder) Collect(o FTAggregateCollect) *AggregateBuilder { + n := len(b.options.Steps) + if n == 0 || b.options.Steps[n-1].GroupBy == nil { + b.setErr(fmt.Errorf("FT.AGGREGATE: Collect must follow a GroupBy step")) + return b + } + reducer, err := NewCollectReducer(o) + if err != nil { + b.setErr(err) + return b + } + g := b.options.Steps[n-1].GroupBy + g.Reduce = append(g.Reduce, reducer) + return b +} + // SortBy adds SORTBY ASC|DESC. Consecutive SortBy calls (with no // other step in between) are merged into a single SORTBY clause so fields // act as tiebreakers. A SortBy call after a non-SortBy step starts a new diff --git a/backend/vendor/github.com/redis/go-redis/v9/search_collect.go b/backend/vendor/github.com/redis/go-redis/v9/search_collect.go new file mode 100644 index 00000000..48389a0c --- /dev/null +++ b/backend/vendor/github.com/redis/go-redis/v9/search_collect.go @@ -0,0 +1,235 @@ +package redis + +import ( + "fmt" + "strings" +) + +// ---------------------- +// FT.AGGREGATE COLLECT reducer +// ---------------------- +// +// COLLECT is a GROUPBY reducer for FT.AGGREGATE (Redis 8.8+, gated behind +// search-enable-unstable-features). Within each group it projects a chosen +// set of fields from every row, optionally deduplicates, sorts, and limits +// them, and emits the result as an array of per-entry maps under the reducer +// alias. +// +// COLLECT is not a standalone command; it is a REDUCE clause inside +// FT.AGGREGATE. The helpers below assemble the reducer token list and compute +// its argument count, so callers do not have to hand-write FIELDS/SORTBY/LIMIT +// tokens or remember to @-prefix every name. + +// FTAggregateCollect describes a COLLECT reducer. It is rendered into a +// standard FTAggregateReducer via NewCollectReducer, or appended to a builder +// via AggregateBuilder.Collect. +// +// Field and sort names may be supplied with or without a leading "@"; each is +// normalized to a single "@" on the wire. Output map keys returned by +// the server are the bare names (see AggregateRow.Collect). +type FTAggregateCollect struct { + // FieldsAll emits FIELDS *, projecting every field present in the + // pipeline at the COLLECT stage. It is not a whole-document fetch; pair + // it with an upstream LOAD * to collect complete documents. FieldsAll + // takes precedence over Fields when both are set. + FieldsAll bool + + // Fields is the explicit list of fields to project (FIELDS @f ...). + // Ignored when FieldsAll is true. Exactly one of FieldsAll or a non-empty + // Fields must be set. + Fields []string + + // Distinct emits DISTINCT, deduplicating entries with identical projected + // fields. + // + // NOTE: DISTINCT is specified by the product but not yet implemented by + // the server. Sending it currently produces a server error. The option is + // kept for forward compatibility; leave it false unless the target server + // supports it. + Distinct bool + + // SortBy orders entries within each group. Direction defaults to ASC when + // neither Asc nor Desc is set. With Limit, SORTBY acts as a top-N + // selection. Reuses FTAggregateSortBy for consistency with the rest of the + // aggregate API. + SortBy []FTAggregateSortBy + + // Limit returns at most Count entries per group after skipping Offset. + // nil means no LIMIT clause (distinct from LIMIT 0 0). + Limit *FTAggregateCollectLimit + + // As sets the reducer output column name (AS ). It is emitted + // outside the reducer argument count. + As string +} + +// FTAggregateCollectLimit is the LIMIT clause of a COLLECT +// reducer. Numeric bounds are enforced by the server, not the client. +type FTAggregateCollectLimit struct { + Offset int + Count int +} + +// ensureAtPrefix normalizes a field or sort name to exactly one leading "@", +// collapsing any number of leading "@" (including none) to a single prefix. +func ensureAtPrefix(name string) string { + return "@" + strings.TrimLeft(name, "@") +} + +// buildCollectArgs renders a FTAggregateCollect into the reducer argument +// token list (everything after "REDUCE COLLECT ", excluding AS ). +// The serializer computes as len(args), which matches the COLLECT +// contract: narg counts every FIELDS/DISTINCT/SORTBY/LIMIT token. +func buildCollectArgs(o FTAggregateCollect) ([]interface{}, error) { + args := make([]interface{}, 0, 8) + + // FIELDS (required): either * or a counted list of @-names. + switch { + case o.FieldsAll: + args = append(args, "FIELDS", "*") + case len(o.Fields) > 0: + args = append(args, "FIELDS", len(o.Fields)) + for _, f := range o.Fields { + if strings.TrimLeft(f, "@") == "" { + return nil, fmt.Errorf("redis: FT.AGGREGATE COLLECT: empty field name in Fields") + } + args = append(args, ensureAtPrefix(f)) + } + default: + return nil, fmt.Errorf("redis: FT.AGGREGATE COLLECT requires FieldsAll or a non-empty Fields list") + } + + // DISTINCT (optional, forward-compatible). + if o.Distinct { + args = append(args, "DISTINCT") + } + + // SORTBY (optional). sort_narg counts each field plus its optional + // direction token. + if len(o.SortBy) > 0 { + sortTokens := make([]interface{}, 0, len(o.SortBy)*2) + for _, s := range o.SortBy { + if strings.TrimLeft(s.FieldName, "@") == "" { + return nil, fmt.Errorf("redis: FT.AGGREGATE COLLECT: empty field name in SortBy") + } + if s.Asc && s.Desc { + return nil, fmt.Errorf("redis: FT.AGGREGATE COLLECT: ASC and DESC are mutually exclusive") + } + sortTokens = append(sortTokens, ensureAtPrefix(s.FieldName)) + switch { + case s.Desc: + sortTokens = append(sortTokens, "DESC") + case s.Asc: + sortTokens = append(sortTokens, "ASC") + // neither set: ASC is the server default; emit nothing. + } + } + args = append(args, "SORTBY", len(sortTokens)) + args = append(args, sortTokens...) + } + + // LIMIT (optional). + if o.Limit != nil { + args = append(args, "LIMIT", o.Limit.Offset, o.Limit.Count) + } + + return args, nil +} + +// NewCollectReducer builds a COLLECT FTAggregateReducer for use with +// FTAggregateOptions.GroupBy[i].Reduce. It normalizes field/sort names and +// computes the argument count automatically. +// +// It returns an error only for local API misuse: a missing FIELDS selector, an +// empty field name (in Fields or SortBy), or a SortBy entry with both Asc and +// Desc set. Numeric bounds and the unstable-features gate are enforced by the +// server and surface unchanged through the command reply. +func NewCollectReducer(o FTAggregateCollect) (FTAggregateReducer, error) { + args, err := buildCollectArgs(o) + if err != nil { + return FTAggregateReducer{}, err + } + return FTAggregateReducer{Reducer: SearchCollect, Args: args, As: o.As}, nil +} + +// ---------------------- +// COLLECT response decoding +// ---------------------- + +// CollectEntry is a single collected row: a sparse map of bare field name to +// value. A field absent from a row is omitted from its entry (no NULL +// placeholder), so entries in the same column may have different key sets. +type CollectEntry = map[string]interface{} + +// CollectColumn is the value stored under a COLLECT reducer alias: the ordered +// list of collected entries for a group. +type CollectColumn = []CollectEntry + +// Collect decodes the COLLECT reducer column stored under alias in this row +// into a uniform CollectColumn, hiding the RESP2/RESP3 representation +// difference (RESP3 entries are maps; RESP2 entries are flat key/value +// arrays). +// +// It returns (nil, nil) when the alias is absent from the row. Entry order is +// preserved as returned by the server; it is meaningful only when the COLLECT +// reducer was given a SORTBY. +func (r AggregateRow) Collect(alias string) (CollectColumn, error) { + v, ok := r.Fields[alias] + if !ok { + return nil, nil + } + return parseCollectValue(v) +} + +// parseCollectValue decodes a raw COLLECT alias value (an array of entries) +// into a CollectColumn. +func parseCollectValue(v interface{}) (CollectColumn, error) { + if v == nil { + return nil, nil + } + arr, ok := v.([]interface{}) + if !ok { + return nil, fmt.Errorf("redis: COLLECT value has type %T, want array of entries", v) + } + out := make(CollectColumn, 0, len(arr)) + for i, e := range arr { + entry, err := parseCollectEntry(e) + if err != nil { + return nil, fmt.Errorf("redis: COLLECT entry %d: %w", i, err) + } + out = append(out, entry) + } + return out, nil +} + +// parseCollectEntry decodes a single collected entry from either the RESP3 +// map form or the RESP2 flat key/value array form into a CollectEntry. Keys +// are passed through as-is: the server already returns them without the "@" +// prefix. +func parseCollectEntry(e interface{}) (CollectEntry, error) { + switch m := e.(type) { + case map[interface{}]interface{}: // RESP3 + out := make(CollectEntry, len(m)) + for k, val := range m { + out[fmt.Sprint(k)] = val + } + return out, nil + case map[string]interface{}: // already string-keyed + return m, nil + case []interface{}: // RESP2 flat [field, value, field, value, ...] + if len(m)%2 != 0 { + return nil, fmt.Errorf("odd-length key/value array of length %d", len(m)) + } + out := make(CollectEntry, len(m)/2) + for i := 0; i < len(m); i += 2 { + key, ok := m[i].(string) + if !ok { + key = fmt.Sprint(m[i]) + } + out[key] = m[i+1] + } + return out, nil + default: + return nil, fmt.Errorf("unexpected type %T, want map or key/value array", e) + } +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/search_commands.go b/backend/vendor/github.com/redis/go-redis/v9/search_commands.go index b13aa5be..588a30e7 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/search_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/search_commands.go @@ -18,6 +18,7 @@ type SearchCmdable interface { FTAggregateWithArgs(ctx context.Context, index string, query string, options *FTAggregateOptions) *AggregateCmd FTAliasAdd(ctx context.Context, index string, alias string) *StatusCmd FTAliasDel(ctx context.Context, alias string) *StatusCmd + FTAliasList(ctx context.Context, index string) *StringSliceCmd FTAliasUpdate(ctx context.Context, index string, alias string) *StatusCmd FTAlter(ctx context.Context, index string, skipInitialScan bool, definition []interface{}) *StatusCmd FTConfigGet(ctx context.Context, option string) *MapMapStringInterfaceCmd @@ -107,6 +108,13 @@ type FTHNSWOptions struct { MaxAllowedEdgesPerNode int EFRunTime int Epsilon float64 + // Rerank toggles the exact re-scoring pass over approximate candidates on + // disk-backed HNSW indexes (Redis 8.10+), where the server requires it to + // be set explicitly. Rerank=true emits RERANK TRUE on its own; to emit + // RERANK FALSE, set HasRerank=true with Rerank=false, so that an explicit + // false can be distinguished from unset (omitted). + Rerank bool + HasRerank bool } type FTVamanaOptions struct { @@ -157,6 +165,12 @@ const ( SearchToList SearchFirstValue SearchRandomSample + // SearchCollect is the COLLECT reducer for FT.AGGREGATE. Within each + // GROUPBY group it projects a chosen set of fields from every row and + // emits them as an array of per-entry maps under the reducer alias. + // Requires Redis 8.8+ with unstable features enabled + // (CONFIG SET search-enable-unstable-features yes). + SearchCollect ) func (a SearchAggregator) String() string { @@ -187,6 +201,8 @@ func (a SearchAggregator) String() string { return "FIRST_VALUE" case SearchRandomSample: return "RANDOM_SAMPLE" + case SearchCollect: + return "COLLECT" default: return "" } @@ -418,8 +434,15 @@ type FTHybridVectorExpression struct { VectorParamName string Method FTHybridVectorMethod MethodParams []interface{} - Filter string - YieldScoreAs string + // ShardKRatio controls how many results each shard returns relative to the + // requested KNN K, trading recall for latency in Redis cluster setups. + // Valid range: 0.1 - 1.0. The zero value means "unset" and falls back to + // the server default of 1.0 (no per-shard reduction). Has no effect on + // standalone Redis, and only applies to the KNN method. Requires Redis 8.8+. + // See https://redis.io/docs/latest/develop/ai/search-and-query/query/vector-search/ + ShardKRatio float64 + Filter string + YieldScoreAs string } // FTHybridCombineOptions represents options for result fusion @@ -487,8 +510,10 @@ type FTSynDumpCmd struct { // FTAggregateResult represents the result of an aggregate operation // NOTE: For RESP3 Total is not reliable (before Redis 8.8) type FTAggregateResult struct { - Total int - Rows []AggregateRow + Total int + Rows []AggregateRow + // Warnings holds server warnings for a partial result (search-on-timeout + // return/return-strict). RESP3 only; the fail policy returns an error instead. Warnings []string } @@ -619,8 +644,10 @@ type SpellCheckSuggestion struct { } type FTSearchResult struct { - Total int - Docs []Document + Total int + Docs []Document + // Warnings holds server warnings for a partial result (search-on-timeout + // return/return-strict). RESP3 only; the fail policy returns an error instead. Warnings []string } @@ -936,22 +963,27 @@ func (cmd *AggregateCmd) SetVal(val *FTAggregateResult) { } func (cmd *AggregateCmd) Val() *FTAggregateResult { + cmd.await() return cmd.val } func (cmd *AggregateCmd) Result() (*FTAggregateResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *AggregateCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *AggregateCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } func (cmd *AggregateCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -1268,6 +1300,20 @@ func (c cmdable) FTAliasDel(ctx context.Context, alias string) *StatusCmd { return cmd } +// FTAliasList - Lists all aliases associated with an index. +// The 'index' parameter specifies the index whose aliases are listed; it must +// be the name of an index created with FT.CREATE, not an alias. +// The reply is an unordered collection of alias names, already deduplicated +// by the server; an index with no aliases yields an empty result, not an +// error. Available since Redis 8.10. +// For more information, please refer to the Redis documentation: +// [FT.ALIASLIST]: (https://redis.io/commands/ft.aliaslist/) +func (c cmdable) FTAliasList(ctx context.Context, index string) *StringSliceCmd { + cmd := NewStringSliceCmd(ctx, "FT.ALIASLIST", index) + _ = c(ctx, cmd) + return cmd +} + // FTAliasUpdate - Updates an alias to an index. // The 'index' parameter specifies the index to which the alias is updated, and the 'alias' parameter specifies the alias. // If the alias already exists for a different index, it updates the alias to point to the specified index instead. @@ -1484,6 +1530,13 @@ func (c cmdable) FTCreate(ctx context.Context, index string, options *FTCreateOp if schema.VectorArgs.HNSWOptions.Epsilon > 0 { hnswArgs = append(hnswArgs, "EPSILON", schema.VectorArgs.HNSWOptions.Epsilon) } + if schema.VectorArgs.HNSWOptions.Rerank || schema.VectorArgs.HNSWOptions.HasRerank { + rerank := "FALSE" + if schema.VectorArgs.HNSWOptions.Rerank { + rerank = "TRUE" + } + hnswArgs = append(hnswArgs, "RERANK", rerank) + } args = append(args, len(hnswArgs)) args = append(args, hnswArgs...) } @@ -1564,7 +1617,6 @@ func (c cmdable) FTCreate(ctx context.Context, index string, options *FTCreateOp } if schema.IndexMissing { args = append(args, "INDEXMISSING") - } } cmd := NewStatusCmd(ctx, args...) @@ -2127,6 +2179,7 @@ func newFTInfoCmd(ctx context.Context, args ...interface{}) *FTInfoCmd { } func (cmd *FTInfoCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2135,20 +2188,25 @@ func (cmd *FTInfoCmd) SetVal(val FTInfoResult) { } func (cmd *FTInfoCmd) Result() (FTInfoResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTInfoCmd) Val() FTInfoResult { + cmd.await() return cmd.val } func (cmd *FTInfoCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTInfoCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } + func (cmd *FTInfoCmd) readReply(rd *proto.Reader) (err error) { readType, err := rd.PeekReplyType() if err != nil { @@ -2329,6 +2387,7 @@ func newFTSpellCheckCmd(ctx context.Context, args ...interface{}) *FTSpellCheckC } func (cmd *FTSpellCheckCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2337,18 +2396,22 @@ func (cmd *FTSpellCheckCmd) SetVal(val []SpellCheckResult) { } func (cmd *FTSpellCheckCmd) Result() ([]SpellCheckResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTSpellCheckCmd) Val() []SpellCheckResult { + cmd.await() return cmd.val } func (cmd *FTSpellCheckCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTSpellCheckCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } @@ -2645,6 +2708,7 @@ func newFTSearchCmd(ctx context.Context, options *FTSearchOptions, args ...inter } func (cmd *FTSearchCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2653,18 +2717,22 @@ func (cmd *FTSearchCmd) SetVal(val FTSearchResult) { } func (cmd *FTSearchCmd) Result() (FTSearchResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTSearchCmd) Val() FTSearchResult { + cmd.await() return cmd.val } func (cmd *FTSearchCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTSearchCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } @@ -2890,8 +2958,10 @@ func (cmd *FTSearchCmd) Clone() Cmder { // FTHybridResult represents the result of a hybrid search operation type FTHybridResult struct { - TotalResults int - Results []map[string]interface{} + TotalResults int + Results []map[string]interface{} + // Warnings holds server warnings for a partial result (search-on-timeout + // return/return-strict), on RESP2 and RESP3; the fail policy returns an error. Warnings []string ExecutionTime float64 } @@ -2926,6 +2996,7 @@ func newFTHybridCmd(ctx context.Context, options *FTHybridOptions, args ...inter } func (cmd *FTHybridCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -2934,26 +3005,32 @@ func (cmd *FTHybridCmd) SetVal(val FTHybridResult) { } func (cmd *FTHybridCmd) Result() (FTHybridResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTHybridCmd) CursorResult() (*FTHybridCursorResult, error) { + cmd.await() return cmd.cursorVal, cmd.err } func (cmd *FTHybridCmd) Val() FTHybridResult { + cmd.await() return cmd.val } func (cmd *FTHybridCmd) CursorVal() *FTHybridCursorResult { + cmd.await() return cmd.cursorVal } func (cmd *FTHybridCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTHybridCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } @@ -3034,9 +3111,13 @@ func parseFTHybrid(data []interface{}, withCursor bool) (FTHybridResult, *FTHybr results = append(results, itemMap) } - // Parse warnings (optional field) + // Optional warnings; accept both "warning" (as FT.SEARCH/FT.AGGREGATE) and "warnings". var warnings []string - if warningsData, ok := resultMap["warnings"].([]interface{}); ok { + warningsData, ok := resultMap["warning"].([]interface{}) + if !ok { + warningsData, ok = resultMap["warnings"].([]interface{}) + } + if ok { warnings = make([]string, 0, len(warningsData)) for _, w := range warningsData { if ws, ok := w.(string); ok { @@ -3460,6 +3541,7 @@ func NewFTSynDumpCmd(ctx context.Context, args ...interface{}) *FTSynDumpCmd { } func (cmd *FTSynDumpCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -3468,18 +3550,22 @@ func (cmd *FTSynDumpCmd) SetVal(val []FTSynDumpResult) { } func (cmd *FTSynDumpCmd) Val() []FTSynDumpResult { + cmd.await() return cmd.val } func (cmd *FTSynDumpCmd) Result() ([]FTSynDumpResult, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *FTSynDumpCmd) RawVal() interface{} { + cmd.await() return cmd.rawVal } func (cmd *FTSynDumpCmd) RawResult() (interface{}, error) { + cmd.await() return cmd.rawVal, cmd.err } @@ -3795,6 +3881,22 @@ func (c cmdable) FTHybridWithArgs(ctx context.Context, index string, options *FT } } + // SHARD_K_RATIO applies to the KNN method only (Redis 8.8+, cluster only). + // Zero means "unset" and falls back to the server default of 1.0. + if vectorExpr.ShardKRatio > 0 { + if vectorExpr.Method != "KNN" { + cmd := newFTHybridCmd(ctx, options, args...) + cmd.SetErr(fmt.Errorf("FT.HYBRID: SHARD_K_RATIO requires KNN method")) + return cmd + } + if vectorExpr.ShardKRatio < 0.1 || vectorExpr.ShardKRatio > 1.0 { + cmd := newFTHybridCmd(ctx, options, args...) + cmd.SetErr(fmt.Errorf("FT.HYBRID: SHARD_K_RATIO must be between 0.1 and 1.0")) + return cmd + } + args = append(args, "SHARD_K_RATIO", vectorExpr.ShardKRatio) + } + if vectorExpr.Filter != "" { args = append(args, "FILTER", vectorExpr.Filter) } diff --git a/backend/vendor/github.com/redis/go-redis/v9/sentinel.go b/backend/vendor/github.com/redis/go-redis/v9/sentinel.go index 055b3101..5720fc47 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/sentinel.go +++ b/backend/vendor/github.com/redis/go-redis/v9/sentinel.go @@ -12,11 +12,14 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/redis/go-redis/v9/auth" "github.com/redis/go-redis/v9/internal" + "github.com/redis/go-redis/v9/internal/otel" "github.com/redis/go-redis/v9/internal/pool" + "github.com/redis/go-redis/v9/internal/proto" "github.com/redis/go-redis/v9/maintnotifications" "github.com/redis/go-redis/v9/push" ) @@ -123,6 +126,18 @@ type FailoverOptions struct { // default: 32KiB (32768 bytes) WriteBufferSize int + // PipelineReadBufferSize, PipelineWriteBufferSize and PipelinePoolSize + // configure an optional separate connection pool used for pipelining, with + // its own (typically larger) buffers. See the same-named fields on Options + // for details. The pool is created only when PipelineReadBufferSize or PipelineWriteBufferSize is set (PipelinePoolSize alone does not enable it). + PipelineReadBufferSize int + PipelineWriteBufferSize int + PipelinePoolSize int + + // AutoPipelineOptions is the default config for the client's autopipeliner + // faces. See Options.AutoPipelineOptions. + AutoPipelineOptions *AutoPipelineOptions + PoolFIFO bool PoolSize int @@ -176,7 +191,7 @@ type FailoverOptions struct { // seamlessly. Requires Protocol: 3 (RESP3) for push notifications. // If nil, maintnotifications upgrades are disabled. // (however if Mode is nil, it defaults to "auto" - enable if server supports it) - //MaintNotificationsConfig *maintnotifications.Config + // MaintNotificationsConfig *maintnotifications.Config } func (opt *FailoverOptions) clientOptions() *Options { @@ -202,6 +217,11 @@ func (opt *FailoverOptions) clientOptions() *Options { ReadBufferSize: opt.ReadBufferSize, WriteBufferSize: opt.WriteBufferSize, + PipelineReadBufferSize: opt.PipelineReadBufferSize, + PipelineWriteBufferSize: opt.PipelineWriteBufferSize, + PipelinePoolSize: opt.PipelinePoolSize, + AutoPipelineOptions: opt.AutoPipelineOptions, + DialTimeout: opt.DialTimeout, DialerRetries: opt.DialerRetries, DialerRetryTimeout: opt.DialerRetryTimeout, @@ -318,6 +338,11 @@ func (opt *FailoverOptions) clusterOptions() *ClusterOptions { ReadBufferSize: opt.ReadBufferSize, WriteBufferSize: opt.WriteBufferSize, + PipelineReadBufferSize: opt.PipelineReadBufferSize, + PipelineWriteBufferSize: opt.PipelineWriteBufferSize, + PipelinePoolSize: opt.PipelinePoolSize, + AutoPipelineOptions: opt.AutoPipelineOptions, + DialTimeout: opt.DialTimeout, DialerRetries: opt.DialerRetries, DialerRetryTimeout: opt.DialerRetryTimeout, @@ -536,8 +561,10 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { rdb := &Client{ baseClient: &baseClient{ - opt: opt, - onClose: &onCloseHooks{}, + apClosed: &atomic.Bool{}, + opt: opt, + onClose: &onCloseHooks{}, + himport: newHImportRegistry(), }, } rdb.init() @@ -561,6 +588,40 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { panic(fmt.Errorf("redis: failed to create pubsub pool: %w", err)) } + // Optionally create a separate connection pool for pipelining, with its own + // (typically larger) buffers. Enabled when either pipeline buffer size is set. + if opt.PipelineReadBufferSize > 0 || opt.PipelineWriteBufferSize > 0 { + pipelineOpt := opt.clone() + if opt.PipelineReadBufferSize > 0 { + pipelineOpt.ReadBufferSize = opt.PipelineReadBufferSize + // Same clamp Options.init applies to the main pool: RESP3 push + // parsing needs a minimum read buffer, and a tiny pipeline reader + // would break push-notification handling on pipeline conns. + if pipelineOpt.Protocol == 3 && pipelineOpt.ReadBufferSize < proto.MinRESP3ReadBufferSize { + pipelineOpt.ReadBufferSize = proto.MinRESP3ReadBufferSize + } + } + if opt.PipelineWriteBufferSize > 0 { + pipelineOpt.WriteBufferSize = opt.PipelineWriteBufferSize + } + if opt.PipelinePoolSize > 0 { + pipelineOpt.PoolSize = opt.PipelinePoolSize + } else { + pipelineOpt.PoolSize = 10 // default smaller pool for pipelining + } + rdb.pipelinePoolName = mainPoolName + "_pipeline" + rdb.pipelinePool, err = newConnPool(pipelineOpt, rdb.dialHook, rdb.pipelinePoolName) + if err != nil { + panic(fmt.Errorf("redis: failed to create pipeline connection pool: %w", err)) + } + } + + // Register pools for OTel async gauge metrics, matching NewClient (the + // failover client previously registered none, so pool gauges were silent + // for the identical standalone setup). The pipeline pool is nil when not + // configured. + otel.RegisterPools(rdb.connPool, rdb.pubSubPool, rdb.pipelinePool, opt.Addr) + rdb.onClose.register(onCloseHookIDSentinelFailover, failover.Close) failover.mu.Lock() @@ -570,6 +631,13 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { return cn.RemoteAddr().String() != addr }) } + // Drop stale pipeline-pool connections dialed to the demoted master too; + // otherwise pipelined traffic keeps using the old address after failover. + if pipelinePool, ok := rdb.pipelinePool.(*pool.ConnPool); ok { + _ = pipelinePool.Filter(func(cn *pool.Conn) bool { + return cn.RemoteAddr().String() != addr + }) + } } failover.mu.Unlock() @@ -599,8 +667,8 @@ func masterReplicaDialer( } netDialer := &net.Dialer{ - Timeout: failover.opt.DialTimeout, - KeepAlive: 5 * time.Minute, + Timeout: failover.opt.DialTimeout, + KeepAliveConfig: defaultKeepAliveConfig, } if failover.opt.TLSConfig == nil { return netDialer.DialContext(ctx, network, addr) @@ -625,8 +693,9 @@ func NewSentinelClient(opt *Options) *SentinelClient { opt.init() c := &SentinelClient{ baseClient: &baseClient{ - opt: opt, - onClose: &onCloseHooks{}, + apClosed: &atomic.Bool{}, + opt: opt, + onClose: &onCloseHooks{}, }, } @@ -678,7 +747,7 @@ func (c *SentinelClient) Process(ctx context.Context, cmd Cmder) error { func (c *SentinelClient) pubSub() *PubSub { pubsub := &PubSub{ - opt: c.opt, + opt: c.cloneOpt(), newConn: func(ctx context.Context, addr string, channels []string) (*pool.Conn, error) { cn, err := c.pubSubPool.NewConn(ctx, c.opt.Network, addr, channels) if err != nil { diff --git a/backend/vendor/github.com/redis/go-redis/v9/set_commands.go b/backend/vendor/github.com/redis/go-redis/v9/set_commands.go index 2a465728..94074555 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/set_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/set_commands.go @@ -12,6 +12,7 @@ type SetCmdable interface { SAdd(ctx context.Context, key string, members ...interface{}) *IntCmd SCard(ctx context.Context, key string) *IntCmd SDiff(ctx context.Context, keys ...string) *StringSliceCmd + SDiffCard(ctx context.Context, opts *SDiffCardOptions, keys ...string) *IntCmd SDiffStore(ctx context.Context, destination string, keys ...string) *IntCmd SInter(ctx context.Context, keys ...string) *StringSliceCmd SInterCard(ctx context.Context, limit int64, keys ...string) *IntCmd @@ -28,9 +29,21 @@ type SetCmdable interface { SRem(ctx context.Context, key string, members ...interface{}) *IntCmd SScan(ctx context.Context, key string, cursor uint64, match string, count int64) *ScanCmd SUnion(ctx context.Context, keys ...string) *StringSliceCmd + SUnionCard(ctx context.Context, opts *SUnionCardOptions, keys ...string) *IntCmd SUnionStore(ctx context.Context, destination string, keys ...string) *IntCmd } +// SUnionCardOptions are the options for SUnionCard. +type SUnionCardOptions struct { + Approx bool // use an approximate (HyperLogLog) count. + Limit int64 // cap the result; 0 means no limit. +} + +// SDiffCardOptions are the options for SDiffCard. +type SDiffCardOptions struct { + Limit int64 // cap the result; 0 means no limit. +} + // Returns the number of elements that were added to the set, not including all // the elements already present in the set. // @@ -96,6 +109,30 @@ func (c cmdable) SDiffStore(ctx context.Context, destination string, keys ...str return cmd } +// Returns the cardinality of the difference of the first set and the rest. +// Missing keys are treated as empty sets. +// +// For more information about the command please refer to [SDIFFCARD]. +// +// [SDIFFCARD]: (https://redis.io/docs/latest/commands/sdiffcard/) +func (c cmdable) SDiffCard(ctx context.Context, opts *SDiffCardOptions, keys ...string) *IntCmd { + if opts == nil { + opts = &SDiffCardOptions{} + } + numKeys := len(keys) + args := make([]interface{}, 0, 4+numKeys) + args = append(args, "sdiffcard", numKeys) + for _, key := range keys { + args = append(args, key) + } + args = append(args, "limit", opts.Limit) + cmd := NewIntCmd(ctx, args...) + // Keys start after the numkeys arg: ["sdiffcard", numKeys, key1, ...]. + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + // Returns the members of the set resulting from the intersection of all the given sets. // Keys that do not exist are considered to be empty sets. // With one of the keys being an empty set, the resulting set is also empty. @@ -328,6 +365,33 @@ func (c cmdable) SUnionStore(ctx context.Context, destination string, keys ...st return cmd } +// Returns the cardinality of the union of all the given sets. +// Missing keys are treated as empty sets. +// +// For more information about the command please refer to [SUNIONCARD]. +// +// [SUNIONCARD]: (https://redis.io/docs/latest/commands/sunioncard/) +func (c cmdable) SUnionCard(ctx context.Context, opts *SUnionCardOptions, keys ...string) *IntCmd { + if opts == nil { + opts = &SUnionCardOptions{} + } + numKeys := len(keys) + args := make([]interface{}, 0, 4+numKeys+1) + args = append(args, "sunioncard", numKeys) + for _, key := range keys { + args = append(args, key) + } + if opts.Approx { + args = append(args, "approx") + } + args = append(args, "limit", opts.Limit) + cmd := NewIntCmd(ctx, args...) + // Keys start after the numkeys arg: ["sunioncard", numKeys, key1, ...]. + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + // Incrementally iterates the set elements stored at key. // This is a cursor-based iterator that allows scanning large sets efficiently. // diff --git a/backend/vendor/github.com/redis/go-redis/v9/stream_commands.go b/backend/vendor/github.com/redis/go-redis/v9/stream_commands.go index e2b2a9e2..95531eb1 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/stream_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/stream_commands.go @@ -227,14 +227,16 @@ func (c cmdable) XRevRangeN(ctx context.Context, stream, start, stop string, cou } type XReadArgs struct { - Streams []string // list of streams and ids, e.g. stream1 stream2 id1 id2 - Count int64 - Block time.Duration - ID string + Streams []string // list of streams and ids, e.g. stream1 stream2 id1 id2 + Count int64 + MaxCount int64 // cumulative cap on total entries across all streams (Redis >= 8.10) + MaxSize int64 // soft cumulative cap on total reply size in bytes across all streams (Redis >= 8.10) + Block time.Duration + ID string } func (c cmdable) XRead(ctx context.Context, a *XReadArgs) *XStreamSliceCmd { - args := make([]interface{}, 0, 2*len(a.Streams)+6) + args := make([]interface{}, 0, 2*len(a.Streams)+10) args = append(args, "xread") keyPos := int8(1) @@ -243,6 +245,14 @@ func (c cmdable) XRead(ctx context.Context, a *XReadArgs) *XStreamSliceCmd { args = append(args, a.Count) keyPos += 2 } + if a.MaxCount > 0 { + args = append(args, "maxcount", a.MaxCount) + keyPos += 2 + } + if a.MaxSize > 0 { + args = append(args, "maxsize", a.MaxSize) + keyPos += 2 + } if a.Block >= 0 { args = append(args, "block") args = append(args, int64(a.Block/time.Millisecond)) @@ -322,13 +332,15 @@ type XReadGroupArgs struct { Consumer string Streams []string // list of streams and ids, e.g. stream1 stream2 id1 id2 Count int64 + MaxCount int64 // cumulative cap on total entries across all streams (Redis >= 8.10) + MaxSize int64 // soft cumulative cap on total reply size in bytes across all streams (Redis >= 8.10) Block time.Duration NoAck bool Claim time.Duration // Claim idle pending entries older than this duration } func (c cmdable) XReadGroup(ctx context.Context, a *XReadGroupArgs) *XStreamSliceCmd { - args := make([]interface{}, 0, 10+len(a.Streams)) + args := make([]interface{}, 0, 14+len(a.Streams)) args = append(args, "xreadgroup", "group", a.Group, a.Consumer) keyPos := int8(4) @@ -336,6 +348,14 @@ func (c cmdable) XReadGroup(ctx context.Context, a *XReadGroupArgs) *XStreamSlic args = append(args, "count", a.Count) keyPos += 2 } + if a.MaxCount > 0 { + args = append(args, "maxcount", a.MaxCount) + keyPos += 2 + } + if a.MaxSize > 0 { + args = append(args, "maxsize", a.MaxSize) + keyPos += 2 + } if a.Block >= 0 { args = append(args, "block", int64(a.Block/time.Millisecond)) keyPos += 2 @@ -361,8 +381,15 @@ func (c cmdable) XReadGroup(ctx context.Context, a *XReadGroupArgs) *XStreamSlic cmd.SetFirstKeyPos(keyPos) _ = c(ctx, cmd) - // Record stream lag for each message (if command succeeded) - if cmd.Err() == nil { + // Record stream lag for each message (if command succeeded). Gated on the + // result being readable WITHOUT blocking: this command carries a + // read-timeout marker, so on the deferred autopipeline face it is diverted + // and still running when we get here — and the default Block: 0 form can + // wait indefinitely for messages, so reading the outcome would block the + // submit call instead of returning a future (review finding by codex on + // #3942). Skipped for a submission that has not executed yet; emitting + // this from the execution path is a follow-up in the OTel wiring. + if otel.Enabled() && cmd.resultReady() && cmd.rawErr() == nil { streams := cmd.Val() for _, stream := range streams { for _, msg := range stream.Messages { diff --git a/backend/vendor/github.com/redis/go-redis/v9/string_commands.go b/backend/vendor/github.com/redis/go-redis/v9/string_commands.go index 6731c09f..3c50715b 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/string_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/string_commands.go @@ -194,6 +194,12 @@ func (c cmdable) GetDel(ctx context.Context, key string) *StringCmd { // (including redis.Nil when the key does not exist) via Err(). If buf is too // small to hold the value, Err() returns a "buffer too small" error. // +// Nothing is ever written past len(buf). When len(buf) >= value length + 2, +// the read takes a fast path that pulls the payload and the protocol's +// trailing CRLF in a single socket read, using the two bytes after the +// payload as scratch — size buffers with 2 spare bytes to opt in (see +// example/zerocopy-buffer). +// // This command opts out of automatic retries because partial data from a // failed attempt would already be sitting in the caller's buffer. func (c cmdable) GetToBuffer(ctx context.Context, key string, buf []byte) *ZeroCopyStringCmd { diff --git a/backend/vendor/github.com/redis/go-redis/v9/timeseries_commands.go b/backend/vendor/github.com/redis/go-redis/v9/timeseries_commands.go index db00db80..673e0e57 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/timeseries_commands.go +++ b/backend/vendor/github.com/redis/go-redis/v9/timeseries_commands.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/redis/go-redis/v9/internal/proto" "github.com/redis/go-redis/v9/internal/util" @@ -30,6 +31,8 @@ type TimeseriesCmdable interface { TSInfoWithArgs(ctx context.Context, key string, options *TSInfoOptions) *MapStringInterfaceCmd TSMAdd(ctx context.Context, ktvSlices [][]interface{}) *IntSliceCmd TSQueryIndex(ctx context.Context, filterExpr []string) *StringSliceCmd + TSQueryLabels(ctx context.Context, filterExpr []string) *StringSliceCmd + TSQueryLabelValues(ctx context.Context, label string, filterExpr []string) *StringSliceCmd TSRevRange(ctx context.Context, key string, fromTimestamp int, toTimestamp int) *TSTimestampValueSliceCmd TSRevRangeWithArgs(ctx context.Context, key string, fromTimestamp int, toTimestamp int, options *TSRevRangeOptions) *TSTimestampValueSliceCmd TSRange(ctx context.Context, key string, fromTimestamp int, toTimestamp int) *TSTimestampValueSliceCmd @@ -40,6 +43,27 @@ type TimeseriesCmdable interface { TSMRevRangeWithArgs(ctx context.Context, fromTimestamp int, toTimestamp int, filterExpr []string, options *TSMRevRangeOptions) *MapStringSliceInterfaceCmd TSMGet(ctx context.Context, filters []string) *MapStringSliceInterfaceCmd TSMGetWithArgs(ctx context.Context, filters []string, options *TSMGetOptions) *MapStringSliceInterfaceCmd + TSNRange(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}) *TSNRangePivotRowSliceCmd + TSNRangeWithArgs(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}, options *TSNRangeOptions) *TSNRangePivotRowSliceCmd + TSNRevRange(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}) *TSNRangePivotRowSliceCmd + TSNRevRangeWithArgs(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}, options *TSNRevRangeOptions) *TSNRangePivotRowSliceCmd + TSRead(ctx context.Context, key string, timestamp interface{}) *TSTimestampValueSliceCmd + TSReadWithArgs(ctx context.Context, key string, timestamp interface{}, options *TSReadOptions) *TSTimestampValueSliceCmd +} + +// TS.READ timestamp cursor sentinels. +const ( + TSReadEarliest = "-" // read from the earliest sample + TSReadLatest = "+" // latest sample, inclusive + TSReadNew = "$" // only samples added after the call +) + +// TSReadOptions holds the optional TS.READ arguments. +type TSReadOptions struct { + Block bool // wait for samples (emits the BLOCK group) + Timeout time.Duration // max wait; 0 blocks indefinitely + MinCount int // unblock threshold; defaults to 1 + MaxCount int // reply cap; 0 is unlimited } type TSOptions struct { @@ -145,6 +169,7 @@ func (a Aggregator) String() string { var ( errTSMultiAggregationGroupBy = errors.New("redis: GROUPBY is not allowed when multiple aggregators are specified") errTSAggregationConflict = errors.New("redis: setting both Aggregator and Aggregators is not allowed; use Aggregators instead because Aggregator is deprecated") + errTSExcludeEmptyGroupBy = errors.New("redis: EXCLUDEEMPTY is not allowed with GROUPBY") ) func formatAggregationArgs(aggregator Aggregator, aggregators []Aggregator) (string, int, error) { @@ -227,8 +252,10 @@ type TSMRangeOptions struct { BucketDuration int BucketTimestamp interface{} Empty bool - GroupByLabel interface{} - Reducer interface{} + // ExcludeEmpty omits matching series that have no samples. Not allowed with GroupByLabel/Reducer. Redis 8.10+. + ExcludeEmpty bool + GroupByLabel interface{} + Reducer interface{} } type TSMRevRangeOptions struct { @@ -245,8 +272,10 @@ type TSMRevRangeOptions struct { BucketDuration int BucketTimestamp interface{} Empty bool - GroupByLabel interface{} - Reducer interface{} + // ExcludeEmpty omits matching series that have no samples. Not allowed with GroupByLabel/Reducer. Redis 8.10+. + ExcludeEmpty bool + GroupByLabel interface{} + Reducer interface{} } type TSMGetOptions struct { @@ -255,6 +284,36 @@ type TSMGetOptions struct { SelectedLabels []interface{} } +type TSNRangeOptions struct { + Latest bool + FilterByTS []int + FilterByValue []float64 // exactly two elements: [min, max] + Count int + Align interface{} + // Aggregators holds exactly one aggregator spec per key. Each spec lists one or + // more aggregators applied to that key and is sent as a single comma-joined token + // (e.g. {{Min, Max}, {Sum}} -> AGGREGATION MIN,MAX SUM ). + Aggregators [][]Aggregator + BucketDuration int + BucketTimestamp interface{} + Empty bool +} + +type TSNRevRangeOptions struct { + Latest bool + FilterByTS []int + FilterByValue []float64 // exactly two elements: [min, max] + Count int + Align interface{} + // Aggregators holds exactly one aggregator spec per key. Each spec lists one or + // more aggregators applied to that key and is sent as a single comma-joined token + // (e.g. {{Min, Max}, {Sum}} -> AGGREGATION MIN,MAX SUM ). + Aggregators [][]Aggregator + BucketDuration int + BucketTimestamp interface{} + Empty bool +} + // TSAdd - Adds one or more observations to a t-digest sketch. // For more information - https://redis.io/commands/ts.add/ func (c cmdable) TSAdd(ctx context.Context, key string, timestamp interface{}, value float64) *IntCmd { @@ -563,6 +622,7 @@ func newTSTimestampValueCmd(ctx context.Context, args ...interface{}) *TSTimesta } func (cmd *TSTimestampValueCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -571,10 +631,12 @@ func (cmd *TSTimestampValueCmd) SetVal(val TSTimestampValue) { } func (cmd *TSTimestampValueCmd) Result() (TSTimestampValue, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *TSTimestampValueCmd) Val() TSTimestampValue { + cmd.await() return cmd.val } @@ -666,6 +728,53 @@ func (c cmdable) TSQueryIndex(ctx context.Context, filterExpr []string) *StringS return cmd } +// TSQueryLabels - Returns the set of label names present on the time series +// matching the filter expressions. Passing no filter expressions queries all +// indexed series. The reply is unordered and already deduplicated by the +// server; it includes the label names used in the filter itself, and an +// empty reply is a valid result, not an error. +// filterExpr uses the same filter language as TSQueryIndex and is passed to +// the server verbatim. Available since Redis 8.10. +// For more information - https://redis.io/commands/ts.querylabels/ +func (c cmdable) TSQueryLabels(ctx context.Context, filterExpr []string) *StringSliceCmd { + args := []interface{}{"TS.QUERYLABELS", "LABELS"} + args = appendTSFilter(args, filterExpr) + cmd := NewStringSliceCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// TSQueryLabelValues - Returns the set of values assigned to the given label +// name across the time series matching the filter expressions. Passing no +// filter expressions queries all indexed series. The label name is matched +// byte-exactly; a label present on no matching series yields an empty reply, +// not an error. The reply is unordered and already deduplicated by the +// server. +// filterExpr uses the same filter language as TSQueryIndex and is passed to +// the server verbatim. Available since Redis 8.10. +// For more information - https://redis.io/commands/ts.querylabels/ +func (c cmdable) TSQueryLabelValues(ctx context.Context, label string, filterExpr []string) *StringSliceCmd { + args := []interface{}{"TS.QUERYLABELS", "VALUES", label} + args = appendTSFilter(args, filterExpr) + cmd := NewStringSliceCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// appendTSFilter appends the FILTER token followed by the filter expressions, +// or nothing when no expressions are given: the server rejects a bare FILTER +// token, and omitting it is the documented way to query all indexed series. +func appendTSFilter(args []interface{}, filterExpr []string) []interface{} { + if len(filterExpr) == 0 { + return args + } + args = append(args, "FILTER") + for _, f := range filterExpr { + args = append(args, f) + } + return args +} + // TSRevRange - Returns a range of samples from a time-series key in reverse order. // For more information - https://redis.io/commands/ts.revrange/ func (c cmdable) TSRevRange(ctx context.Context, key string, fromTimestamp int, toTimestamp int) *TSTimestampValueSliceCmd { @@ -790,6 +899,47 @@ func (c cmdable) TSRangeWithArgs(ctx context.Context, key string, fromTimestamp return cmd } +// TSRead - Returns samples at or after timestamp, in ascending order. +// timestamp is a non-negative Unix-ms integer or a sentinel (TSReadEarliest, +// TSReadLatest, TSReadNew). +// For more information - https://redis.io/commands/ts.read/ +func (c cmdable) TSRead(ctx context.Context, key string, timestamp interface{}) *TSTimestampValueSliceCmd { + args := []interface{}{"TS.READ", key, timestamp} + cmd := newTSTimestampValueSliceCmd(ctx, args...) + _ = c(ctx, cmd) + return cmd +} + +// TSReadWithArgs - TS.READ with the optional BLOCK and MAX_COUNT groups. +// When options.Block is set it waits for options.MinCount samples or until +// options.Timeout elapses. Blocking calls must not be used in a pipeline or MULTI. +// For more information - https://redis.io/commands/ts.read/ +func (c cmdable) TSReadWithArgs(ctx context.Context, key string, timestamp interface{}, options *TSReadOptions) *TSTimestampValueSliceCmd { + args := []interface{}{"TS.READ", key, timestamp} + blocking := false + var blockTimeout time.Duration + if options != nil { + if options.Block { + blocking = true + blockTimeout = options.Timeout + minCount := options.MinCount + if minCount <= 0 { + minCount = 1 + } + args = append(args, "BLOCK", formatMs(ctx, options.Timeout), minCount) + } + if options.MaxCount != 0 { + args = append(args, "MAX_COUNT", options.MaxCount) + } + } + cmd := newTSTimestampValueSliceCmd(ctx, args...) + if blocking { + cmd.setReadTimeout(blockTimeout) + } + _ = c(ctx, cmd) + return cmd +} + type TSTimestampValueSliceCmd struct { baseCmd val []TSTimestampValue @@ -806,6 +956,7 @@ func newTSTimestampValueSliceCmd(ctx context.Context, args ...interface{}) *TSTi } func (cmd *TSTimestampValueSliceCmd) String() string { + cmd.await() return cmdString(cmd, cmd.val) } @@ -814,10 +965,12 @@ func (cmd *TSTimestampValueSliceCmd) SetVal(val []TSTimestampValue) { } func (cmd *TSTimestampValueSliceCmd) Result() ([]TSTimestampValue, error) { + cmd.await() return cmd.val, cmd.err } func (cmd *TSTimestampValueSliceCmd) Val() []TSTimestampValue { + cmd.await() return cmd.val } @@ -896,11 +1049,8 @@ func (c cmdable) TSMRange(ctx context.Context, fromTimestamp int, toTimestamp in return cmd } -// TSMRangeWithArgs - Returns a range of samples from multiple time-series keys with additional options. -// This function allows for specifying additional options such as: -// Latest, FilterByTS, FilterByValue, WithLabels, SelectedLabels, -// Count, Align, Aggregator, BucketDuration, BucketTimestamp, -// Empty, GroupByLabel and Reducer. +// TSMRangeWithArgs - Returns a range of samples from multiple time-series keys. +// Options are set via TSMRangeOptions. // For more information - https://redis.io/commands/ts.mrange/ func (c cmdable) TSMRangeWithArgs(ctx context.Context, fromTimestamp int, toTimestamp int, filterExpr []string, options *TSMRangeOptions) *MapStringSliceInterfaceCmd { args := []interface{}{"TS.MRANGE", fromTimestamp, toTimestamp} @@ -953,12 +1103,20 @@ func (c cmdable) TSMRangeWithArgs(ctx context.Context, fromTimestamp int, toTime if options.Empty { args = append(args, "EMPTY") } + if options.ExcludeEmpty { + args = append(args, "EXCLUDEEMPTY") + } } args = append(args, "FILTER") for _, f := range filterExpr { args = append(args, f) } if options != nil { + if options.ExcludeEmpty && (options.GroupByLabel != nil || options.Reducer != nil) { + cmd := NewMapStringSliceInterfaceCmd(ctx, args...) + cmd.SetErr(errTSExcludeEmptyGroupBy) + return cmd + } if multiAggregationCount > 1 && (options.GroupByLabel != nil || options.Reducer != nil) { cmd := NewMapStringSliceInterfaceCmd(ctx, args...) cmd.SetErr(errTSMultiAggregationGroupBy) @@ -988,11 +1146,8 @@ func (c cmdable) TSMRevRange(ctx context.Context, fromTimestamp int, toTimestamp return cmd } -// TSMRevRangeWithArgs - Returns a range of samples from multiple time-series keys in reverse order with additional options. -// This function allows for specifying additional options such as: -// Latest, FilterByTS, FilterByValue, WithLabels, SelectedLabels, -// Count, Align, Aggregator, BucketDuration, BucketTimestamp, -// Empty, GroupByLabel and Reducer. +// TSMRevRangeWithArgs - Returns a range of samples from multiple time-series keys in reverse order. +// Options are set via TSMRevRangeOptions. // For more information - https://redis.io/commands/ts.mrevrange/ func (c cmdable) TSMRevRangeWithArgs(ctx context.Context, fromTimestamp int, toTimestamp int, filterExpr []string, options *TSMRevRangeOptions) *MapStringSliceInterfaceCmd { args := []interface{}{"TS.MREVRANGE", fromTimestamp, toTimestamp} @@ -1045,12 +1200,20 @@ func (c cmdable) TSMRevRangeWithArgs(ctx context.Context, fromTimestamp int, toT if options.Empty { args = append(args, "EMPTY") } + if options.ExcludeEmpty { + args = append(args, "EXCLUDEEMPTY") + } } args = append(args, "FILTER") for _, f := range filterExpr { args = append(args, f) } if options != nil { + if options.ExcludeEmpty && (options.GroupByLabel != nil || options.Reducer != nil) { + cmd := NewMapStringSliceInterfaceCmd(ctx, args...) + cmd.SetErr(errTSExcludeEmptyGroupBy) + return cmd + } if multiAggregationCount > 1 && (options.GroupByLabel != nil || options.Reducer != nil) { cmd := NewMapStringSliceInterfaceCmd(ctx, args...) cmd.SetErr(errTSMultiAggregationGroupBy) @@ -1106,3 +1269,279 @@ func (c cmdable) TSMGetWithArgs(ctx context.Context, filters []string, options * _ = c(ctx, cmd) return cmd } + +// TSNRangePivotRow represents a single row in the pivot response from TS.NRANGE / TS.NREVRANGE. +// Timestamp is the row's timestamp. Without aggregation, Values holds one float64 per input key +// in input-key order. With aggregation, Values holds one float64 per requested (key, aggregator) +// pair, flattened in input-key order with each key's aggregators in spec order. +// Missing samples and missing aggregation buckets are represented as NaN. +type TSNRangePivotRow struct { + Timestamp int64 + Values []float64 +} + +type TSNRangePivotRowSliceCmd struct { + baseCmd + val []TSNRangePivotRow +} + +func newTSNRangePivotRowSliceCmd(ctx context.Context, args ...interface{}) *TSNRangePivotRowSliceCmd { + return &TSNRangePivotRowSliceCmd{ + baseCmd: baseCmd{ + ctx: ctx, + args: args, + cmdType: CmdTypeTSNRangePivotRowSlice, + }, + } +} + +func (cmd *TSNRangePivotRowSliceCmd) String() string { + cmd.await() + return cmdString(cmd, cmd.val) +} + +func (cmd *TSNRangePivotRowSliceCmd) SetVal(val []TSNRangePivotRow) { + cmd.val = val +} + +func (cmd *TSNRangePivotRowSliceCmd) Result() ([]TSNRangePivotRow, error) { + cmd.await() + return cmd.val, cmd.err +} + +func (cmd *TSNRangePivotRowSliceCmd) Val() []TSNRangePivotRow { + cmd.await() + return cmd.val +} + +func (cmd *TSNRangePivotRowSliceCmd) readReply(rd *proto.Reader) error { + n, err := rd.ReadArrayLen() + if err != nil { + return err + } + cmd.val = make([]TSNRangePivotRow, n) + for i := 0; i < n; i++ { + // Each row is a 2-element array: [timestamp, [value_0, value_1, ...]] + if _, err = rd.ReadArrayLen(); err != nil { + return err + } + timestamp, err := rd.ReadInt() + if err != nil { + return err + } + cmd.val[i].Timestamp = timestamp + + valCount, err := rd.ReadArrayLen() + if err != nil { + return err + } + cmd.val[i].Values = make([]float64, valCount) + for j := 0; j < valCount; j++ { + s, err := rd.ReadString() + if err != nil { + return err + } + cmd.val[i].Values[j], err = util.ParseStringToFloat(s) + if err != nil { + return err + } + } + } + return nil +} + +func (cmd *TSNRangePivotRowSliceCmd) Clone() Cmder { + var val []TSNRangePivotRow + if cmd.val != nil { + val = make([]TSNRangePivotRow, len(cmd.val)) + copy(val, cmd.val) + for i := range cmd.val { + if cmd.val[i].Values != nil { + val[i].Values = make([]float64, len(cmd.val[i].Values)) + copy(val[i].Values, cmd.val[i].Values) + } + } + } + return &TSNRangePivotRowSliceCmd{ + baseCmd: cmd.cloneBaseCmd(), + val: val, + } +} + +// buildNRangeAggregationArgs validates and returns one aggregator spec string per key for +// TS.NRANGE / TS.NREVRANGE. The number of specs must equal the number of keys. Each spec +// lists one or more aggregators for its key and is emitted as a single comma-joined wire +// token; specs for different keys are separate wire tokens. +func buildNRangeAggregationArgs(keys []string, aggregators [][]Aggregator) ([]string, error) { + if len(aggregators) != len(keys) { + return nil, fmt.Errorf("redis: TS.NRANGE/TS.NREVRANGE requires exactly %d aggregator spec(s), got %d", len(keys), len(aggregators)) + } + parts := make([]string, len(aggregators)) + for i, spec := range aggregators { + if len(spec) == 0 { + return nil, fmt.Errorf("redis: empty timeseries aggregator spec at index %d", i) + } + names := make([]string, len(spec)) + for j, agg := range spec { + if agg == Invalid { + return nil, fmt.Errorf("redis: invalid timeseries aggregator at index %d[%d]: Invalid (%d)", i, j, agg) + } + s := agg.String() + if s == "" { + return nil, fmt.Errorf("redis: invalid timeseries aggregator at index %d[%d]: %d", i, j, agg) + } + names[j] = s + } + parts[i] = strings.Join(names, ",") + } + return parts, nil +} + +// appendNRangeOptions appends optional TS.NRANGE / TS.NREVRANGE arguments to args. +func appendNRangeOptions( + args []interface{}, + keys []string, + latest bool, + filterByTS []int, + filterByValue []float64, + count int, + align interface{}, + aggregators [][]Aggregator, + bucketDuration int, + bucketTimestamp interface{}, + empty bool, +) ([]interface{}, error) { + if latest { + args = append(args, "LATEST") + } + if len(filterByTS) > 0 { + args = append(args, "FILTER_BY_TS") + for _, ts := range filterByTS { + args = append(args, ts) + } + } + if len(filterByValue) > 0 { + if len(filterByValue) != 2 { + return args, fmt.Errorf("redis: FILTER_BY_VALUE requires exactly 2 elements [min, max], got %d", len(filterByValue)) + } + args = append(args, "FILTER_BY_VALUE", filterByValue[0], filterByValue[1]) + } + if count != 0 { + args = append(args, "COUNT", count) + } + if align != nil { + args = append(args, "ALIGN", align) + } + if len(aggregators) > 0 { + aggParts, err := buildNRangeAggregationArgs(keys, aggregators) + if err != nil { + return args, err + } + args = append(args, "AGGREGATION") + for _, a := range aggParts { + args = append(args, a) + } + if bucketDuration != 0 { + args = append(args, bucketDuration) + } + if bucketTimestamp != nil { + args = append(args, "BUCKETTIMESTAMP", bucketTimestamp) + } + if empty { + args = append(args, "EMPTY") + } + } + return args, nil +} + +// TSNRange - Queries multiple time-series keys and returns a pivot response in forward (ascending) order. +// For more information - https://redis.io/commands/ts.nrange/ +func (c cmdable) TSNRange(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}) *TSNRangePivotRowSliceCmd { + args := make([]interface{}, 0, 3+len(keys)) + args = append(args, "TS.NRANGE", len(keys)) + for _, k := range keys { + args = append(args, k) + } + args = append(args, fromTimestamp, toTimestamp) + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + +// TSNRangeWithArgs - Queries multiple time-series keys and returns a pivot response in forward (ascending) order with additional options. +// This function allows for specifying additional options such as: +// Latest, FilterByTS, FilterByValue, Count, Align, Aggregators, BucketDuration, BucketTimestamp and Empty. +// Aggregators must contain exactly one spec per key; each spec lists one or more aggregators +// for its key and is emitted as a single comma-joined wire token. +// For more information - https://redis.io/commands/ts.nrange/ +func (c cmdable) TSNRangeWithArgs(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}, options *TSNRangeOptions) *TSNRangePivotRowSliceCmd { + args := make([]interface{}, 0, 3+len(keys)) + args = append(args, "TS.NRANGE", len(keys)) + for _, k := range keys { + args = append(args, k) + } + args = append(args, fromTimestamp, toTimestamp) + if options != nil { + var err error + args, err = appendNRangeOptions(args, keys, + options.Latest, options.FilterByTS, options.FilterByValue, + options.Count, options.Align, options.Aggregators, + options.BucketDuration, options.BucketTimestamp, options.Empty) + if err != nil { + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetErr(err) + return cmd + } + } + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + +// TSNRevRange - Queries multiple time-series keys and returns a pivot response in reverse (descending) order. +// For more information - https://redis.io/commands/ts.nrevrange/ +func (c cmdable) TSNRevRange(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}) *TSNRangePivotRowSliceCmd { + args := make([]interface{}, 0, 3+len(keys)) + args = append(args, "TS.NREVRANGE", len(keys)) + for _, k := range keys { + args = append(args, k) + } + args = append(args, fromTimestamp, toTimestamp) + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} + +// TSNRevRangeWithArgs - Queries multiple time-series keys and returns a pivot response in reverse (descending) order with additional options. +// This function allows for specifying additional options such as: +// Latest, FilterByTS, FilterByValue, Count, Align, Aggregators, BucketDuration, BucketTimestamp and Empty. +// Aggregators must contain exactly one spec per key; each spec lists one or more aggregators +// for its key and is emitted as a single comma-joined wire token. +// For more information - https://redis.io/commands/ts.nrevrange/ +func (c cmdable) TSNRevRangeWithArgs(ctx context.Context, keys []string, fromTimestamp interface{}, toTimestamp interface{}, options *TSNRevRangeOptions) *TSNRangePivotRowSliceCmd { + args := make([]interface{}, 0, 3+len(keys)) + args = append(args, "TS.NREVRANGE", len(keys)) + for _, k := range keys { + args = append(args, k) + } + args = append(args, fromTimestamp, toTimestamp) + if options != nil { + var err error + args, err = appendNRangeOptions(args, keys, + options.Latest, options.FilterByTS, options.FilterByValue, + options.Count, options.Align, options.Aggregators, + options.BucketDuration, options.BucketTimestamp, options.Empty) + if err != nil { + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetErr(err) + return cmd + } + } + cmd := newTSNRangePivotRowSliceCmd(ctx, args...) + cmd.SetFirstKeyPos(2) + _ = c(ctx, cmd) + return cmd +} diff --git a/backend/vendor/github.com/redis/go-redis/v9/tx.go b/backend/vendor/github.com/redis/go-redis/v9/tx.go index 179230e3..0e4e33da 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/tx.go +++ b/backend/vendor/github.com/redis/go-redis/v9/tx.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "github.com/redis/go-redis/v9/internal/pool" "github.com/redis/go-redis/v9/internal/proto" ) @@ -44,10 +43,20 @@ func (c *Client) newTx() *Tx { tx := Tx{ baseClient: baseClient{ opt: c.cloneOpt(), // Clone options under optLock to avoid race with initConn - connPool: pool.NewStickyConnPool(c.connPool), + connPool: c.baseClient.newStickyConnPool(), hooksMixin: c.hooksMixin.clone(), pushProcessor: c.pushProcessor, // Copy push processor from parent client onClose: &onCloseHooks{}, + // Share the HIMPORT fieldset registry: the sticky pool borrows + // connections from the parent client's pool, so fieldsets + // prepared on them stay valid after the connections are + // returned. + himport: c.himport, + // Carry the shared eviction hook (not csc: a sticky Tx must not serve + // cached reads) so close/reinit hooks on a Watch-initialized conn still + // evict from the parent cache. + cscPoolHook: c.cscPoolHook, + cscActive: c.cscActive, }, } tx.init() diff --git a/backend/vendor/github.com/redis/go-redis/v9/universal.go b/backend/vendor/github.com/redis/go-redis/v9/universal.go index b623460c..40f72ac5 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/universal.go +++ b/backend/vendor/github.com/redis/go-redis/v9/universal.go @@ -149,8 +149,31 @@ type UniversalOptions struct { // IsClusterMode can be used when only one Addrs is provided (e.g. Elasticache supports setting up cluster mode with configuration endpoint). IsClusterMode bool + // AutoPipelineOptions is the default config for the client's + // autopipeliner faces (AutoPipeline / AsyncAutoPipeline), applied when + // they are called without explicit options. See Options.AutoPipelineOptions. + AutoPipelineOptions *AutoPipelineOptions + // MaintNotificationsConfig provides configuration for maintnotifications upgrades. MaintNotificationsConfig *maintnotifications.Config + + // ClientSideCacheConfig enables client-side caching when NewUniversalClient + // selects a standalone Client. See Options.ClientSideCacheConfig. + // + // Experimental: this API may change in a minor release. + ClientSideCacheConfig *ClientSideCacheConfig + + // ClientSideCache supplies an explicit cache when NewUniversalClient selects + // a standalone Client. See Options.ClientSideCache. + // + // Experimental: this API may change in a minor release. + ClientSideCache Cache + + // ClientSideCacheStrategy selects the standalone client's invalidation + // strategy. See Options.ClientSideCacheStrategy. + // + // Experimental: this API may change in a minor release. + ClientSideCacheStrategy CSCStrategy } // Cluster returns cluster options created from the universal options. @@ -208,6 +231,7 @@ func (o *UniversalOptions) Cluster() *ClusterOptions { DisableIdentity: o.DisableIdentity, DisableIndentity: o.DisableIndentity, IdentitySuffix: o.IdentitySuffix, + AutoPipelineOptions: o.AutoPipelineOptions, FailingTimeoutSeconds: o.FailingTimeoutSeconds, UnstableResp3: o.UnstableResp3, PushNotificationProcessor: o.PushNotificationProcessor, @@ -276,6 +300,7 @@ func (o *UniversalOptions) Failover() *FailoverOptions { DisableIdentity: o.DisableIdentity, DisableIndentity: o.DisableIndentity, IdentitySuffix: o.IdentitySuffix, + AutoPipelineOptions: o.AutoPipelineOptions, UnstableResp3: o.UnstableResp3, PushNotificationProcessor: o.PushNotificationProcessor, // Note: MaintNotificationsConfig not supported for FailoverOptions @@ -334,9 +359,13 @@ func (o *UniversalOptions) Simple() *Options { DisableIdentity: o.DisableIdentity, DisableIndentity: o.DisableIndentity, IdentitySuffix: o.IdentitySuffix, + AutoPipelineOptions: o.AutoPipelineOptions, UnstableResp3: o.UnstableResp3, PushNotificationProcessor: o.PushNotificationProcessor, MaintNotificationsConfig: o.MaintNotificationsConfig, + ClientSideCacheConfig: o.ClientSideCacheConfig, + ClientSideCache: o.ClientSideCache, + ClientSideCacheStrategy: o.ClientSideCacheStrategy, } } @@ -352,6 +381,15 @@ type UniversalClient interface { Watch(ctx context.Context, fn func(*Tx) error, keys ...string) error Do(ctx context.Context, args ...interface{}) *Cmd Process(ctx context.Context, cmd Cmder) error + // AutoPipeline / AsyncAutoPipeline return an AutoPipeliner for the concrete + // client. Supported on *Client (including sentinel-backed failover clients) + // and *ClusterClient; *Ring returns an error (not supported). + // + // EXPERIMENTAL: this API is subject to change, use with caution. + AutoPipeline() (*AutoPipeliner, error) + AutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) + AsyncAutoPipeline() (*AutoPipeliner, error) + AsyncAutoPipelineWithOptions(config *AutoPipelineOptions) (*AutoPipeliner, error) Subscribe(ctx context.Context, channels ...string) *PubSub PSubscribe(ctx context.Context, channels ...string) *PubSub SSubscribe(ctx context.Context, channels ...string) *PubSub @@ -363,6 +401,9 @@ var ( _ UniversalClient = (*Client)(nil) _ UniversalClient = (*ClusterClient)(nil) _ UniversalClient = (*Ring)(nil) + // AutoPipeliner is a drop-in for the real clients; non-data operations + // delegate to the underlying client. + _ UniversalClient = (*AutoPipeliner)(nil) ) // NewUniversalClient returns a new multi client. The type of the returned client depends diff --git a/backend/vendor/github.com/redis/go-redis/v9/version.go b/backend/vendor/github.com/redis/go-redis/v9/version.go index c6cabc69..9ce635e1 100644 --- a/backend/vendor/github.com/redis/go-redis/v9/version.go +++ b/backend/vendor/github.com/redis/go-redis/v9/version.go @@ -2,5 +2,5 @@ package redis // Version is the current release version. func Version() string { - return "9.21.0" + return "9.22.0" } diff --git a/backend/vendor/github.com/slack-go/slack/CHANGELOG.md b/backend/vendor/github.com/slack-go/slack/CHANGELOG.md index 62c52147..f904dd67 100644 --- a/backend/vendor/github.com/slack-go/slack/CHANGELOG.md +++ b/backend/vendor/github.com/slack-go/slack/CHANGELOG.md @@ -7,7 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.27.0] +## [0.29.0] - 2026-08-15 + +### Fixed + +- `slackevents`: `AppMentionEvent` and `MessageEvent` now expose the Data Access API action + token through a new `ActionToken` field, which reads `action_token` from the event object + itself. Slack sends the token there on `app_mention` and `message` events, but both types + only modelled it nested inside `assistant_thread`, so the token was silently dropped and + bot-token calls to `assistant.search.context` failed with `invalid_action_token`. The + existing `AssistantThread` field is unchanged (#1577, #1580). + +## [0.28.0] - 2026-08-15 + +### Added + +- Block Kit: Add support for [`container`](https://docs.slack.dev/reference/block-kit/blocks/container-block/) block through `ContainerBlock`, with a `NewContainerBlock` constructor, fluent `With*` builders (`WithTitle`, `WithRichTextTitle`, `WithSubtitle`, `WithIcon`, `WithWidth`, `WithCollapsible`, `WithHeaderDivider`, `WithBlockID`), an `AddChildBlock` helper and a `Validate` method (#1574). +- `PostMessageWithResponse` and `PostMessageWithResponseContext` return the full `Message` object from the `chat.postMessage` response alongside the channel and timestamp, giving access to response-only fields such as `Message.ThreadTimestamp` (#1572). +- `CompleteUploadExternalParameters` and `UploadFileParameters` now take `Channels`, which shares a single uploaded file with up to 100 conversations in one `files.completeUploadExternal` request (#1579). + +## [0.27.0] - 2026-06-27 ### Added @@ -16,7 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Interaction payloads: `Team` now preserves `enterprise_id` and `enterprise_name` when Slack includes Enterprise org details in interaction payload `team` objects. -## [0.26.0] +## [0.26.0] - 2026-06-14 ### Added @@ -29,7 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and the `subtext` field (with a `WithSubtext` builder). `slack_icon` is mutually exclusive with `icon` (#1562). -## [0.25.0] +## [0.25.0] - 2026-06-05 ### Changed @@ -39,7 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `TableBlock.Rows` is now `[][]TableCell` (was `[][]*RichTextBlock`), so `table` blocks no longer drop `raw_text`, `raw_number`, and `null` cells (#1558). -## [0.24.0] +## [0.24.0] - 2026-05-24 ### Added @@ -611,7 +630,9 @@ for details. [#1196]: https://github.com/slack-go/slack/issues/1196 [#1547]: https://github.com/slack-go/slack/pull/1547 -[Unreleased]: https://github.com/slack-go/slack/compare/v0.27.0...HEAD +[Unreleased]: https://github.com/slack-go/slack/compare/v0.29.0...HEAD +[0.29.0]: https://github.com/slack-go/slack/compare/v0.28.0...v0.29.0 +[0.28.0]: https://github.com/slack-go/slack/compare/v0.27.0...v0.28.0 [0.27.0]: https://github.com/slack-go/slack/compare/v0.26.0...v0.27.0 [0.26.0]: https://github.com/slack-go/slack/compare/v0.25.0...v0.26.0 [0.25.0]: https://github.com/slack-go/slack/compare/v0.24.0...v0.25.0 diff --git a/backend/vendor/github.com/slack-go/slack/audit.go b/backend/vendor/github.com/slack-go/slack/audit.go index f258a412..9ec39f73 100644 --- a/backend/vendor/github.com/slack-go/slack/audit.go +++ b/backend/vendor/github.com/slack-go/slack/audit.go @@ -115,7 +115,7 @@ func (api *Client) auditLogsRequest(ctx context.Context, path string, values url return response, response.Err() } -// GetAuditLogs retrieves a page of audit entires according to the parameters given +// GetAuditLogs retrieves a page of audit entries according to the parameters given func (api *Client) GetAuditLogs(params AuditLogParameters) (entries []AuditEntry, nextCursor string, err error) { return api.GetAuditLogsContext(context.Background(), params) } diff --git a/backend/vendor/github.com/slack-go/slack/block.go b/backend/vendor/github.com/slack-go/slack/block.go index fd281e0e..41c1cef4 100644 --- a/backend/vendor/github.com/slack-go/slack/block.go +++ b/backend/vendor/github.com/slack-go/slack/block.go @@ -26,6 +26,7 @@ const ( MBTAlert MessageBlockType = "alert" MBTCard MessageBlockType = "card" MBTCarousel MessageBlockType = "carousel" + MBTContainer MessageBlockType = "container" ) // Block defines an interface all block types should implement diff --git a/backend/vendor/github.com/slack-go/slack/block_container.go b/backend/vendor/github.com/slack-go/slack/block_container.go new file mode 100644 index 00000000..b96c8566 --- /dev/null +++ b/backend/vendor/github.com/slack-go/slack/block_container.go @@ -0,0 +1,182 @@ +package slack + +import "fmt" + +// ContainerWidth controls the rendered width of a ContainerBlock. When unset, +// Slack defaults to ContainerWidthStandard. +type ContainerWidth string + +const ( + ContainerWidthNarrow ContainerWidth = "narrow" + ContainerWidthStandard ContainerWidth = "standard" + ContainerWidthWide ContainerWidth = "wide" + ContainerWidthFull ContainerWidth = "full" +) + +// containerMaxChildBlocks is the maximum number of child blocks Slack permits +// inside a container block. +const containerMaxChildBlocks = 10 + +// ContainerBlock groups a set of child blocks so they render together as a +// single, optionally collapsible, unit with a title, subtitle, and icon. +// +// More Information: https://docs.slack.dev/reference/block-kit/blocks/container-block/ +type ContainerBlock struct { + Type MessageBlockType `json:"type"` + BlockID string `json:"block_id,omitempty"` + // Title is the container heading rendered as a plain_text object. One of + // Title or RichTextTitle is required; RichTextTitle takes precedence when + // both are set. Slack requires a maximum of 150 characters. + Title *TextBlockObject `json:"title,omitempty"` + // RichTextTitle is the container heading rendered as a rich_text block. It + // takes precedence over Title when both are set. + RichTextTitle *RichTextBlock `json:"rich_text_title,omitempty"` + // Subtitle is descriptive text below the title, rendered as a plain_text or + // mrkdwn object. Slack requires a maximum of 150 characters. + Subtitle *TextBlockObject `json:"subtitle,omitempty"` + // Icon is a small image displayed beside the title and subtitle. + Icon *ImageBlockElement `json:"icon,omitempty"` + // Width controls the container width. Slack defaults to standard when unset. + Width ContainerWidth `json:"width,omitempty"` + // IsCollapsible enables the container's collapse control. + IsCollapsible bool `json:"is_collapsible,omitempty"` + // DefaultCollapsed starts the container collapsed. It only applies when + // IsCollapsible is true. + DefaultCollapsed bool `json:"default_collapsed,omitempty"` + // HasHeaderDivider draws a border below the header. Slack only supports it on + // non-collapsible containers. + HasHeaderDivider bool `json:"has_header_divider,omitempty"` + // ChildBlocks are the blocks rendered inside the container. Slack requires 1 + // to 10 blocks: actions, context, divider, file, header, image, input, + // rich_text, section, table, and video blocks are supported. + ChildBlocks Blocks `json:"child_blocks"` +} + +// BlockType returns the type of the block. +func (s ContainerBlock) BlockType() MessageBlockType { + return s.Type +} + +// ID returns the ID of the block. +func (s ContainerBlock) ID() string { + return s.BlockID +} + +// Validate checks whether the block satisfies Slack's documented container +// constraints. +func (s ContainerBlock) Validate() error { + if s.Type != MBTContainer { + return fmt.Errorf("type must be %q", MBTContainer) + } + if s.Title == nil && s.RichTextTitle == nil { + return fmt.Errorf("one of title or rich_text_title is required") + } + if s.Title != nil { + if s.Title.Type != PlainTextType { + return fmt.Errorf("title must be a plain_text object") + } + if runeLen(s.Title.Text) > 150 { + return fmt.Errorf("title cannot be longer than 150 characters") + } + } + if s.Subtitle != nil { + if s.Subtitle.Type != PlainTextType && s.Subtitle.Type != MarkdownType { + return fmt.Errorf("subtitle must be a plain_text or mrkdwn object") + } + if runeLen(s.Subtitle.Text) > 150 { + return fmt.Errorf("subtitle cannot be longer than 150 characters") + } + } + switch s.Width { + case "", ContainerWidthNarrow, ContainerWidthStandard, ContainerWidthWide, ContainerWidthFull: + default: + return fmt.Errorf("width must be one of narrow, standard, wide, or full") + } + if s.Icon != nil { + if runeLen(s.Icon.AltText) > 2000 { + return fmt.Errorf("icon alt_text cannot be longer than 2000 characters") + } + if s.Icon.ImageURL != nil && runeLen(*s.Icon.ImageURL) > 3000 { + return fmt.Errorf("icon image_url cannot be longer than 3000 characters") + } + } + if s.HasHeaderDivider && s.IsCollapsible { + return fmt.Errorf("has_header_divider is only supported on non-collapsible containers") + } + if s.DefaultCollapsed && !s.IsCollapsible { + return fmt.Errorf("default_collapsed requires is_collapsible to be true") + } + if n := len(s.ChildBlocks.BlockSet); n < 1 { + return fmt.Errorf("child_blocks must have at least 1 block") + } else if n > containerMaxChildBlocks { + return fmt.Errorf("child_blocks cannot have more than %d blocks", containerMaxChildBlocks) + } + return nil +} + +// NewContainerBlock returns a new container block wrapping the given child +// blocks. Use the With* methods to set the title, subtitle, and other optional +// fields. +func NewContainerBlock(childBlocks ...Block) *ContainerBlock { + return &ContainerBlock{ + Type: MBTContainer, + ChildBlocks: Blocks{BlockSet: childBlocks}, + } +} + +// WithBlockID sets the block ID for the ContainerBlock. +func (s *ContainerBlock) WithBlockID(blockID string) *ContainerBlock { + s.BlockID = blockID + return s +} + +// WithTitle sets the plain_text title for the ContainerBlock. +func (s *ContainerBlock) WithTitle(title *TextBlockObject) *ContainerBlock { + s.Title = title + return s +} + +// WithRichTextTitle sets the rich_text title for the ContainerBlock. It takes +// precedence over a plain_text title set with WithTitle. +func (s *ContainerBlock) WithRichTextTitle(title *RichTextBlock) *ContainerBlock { + s.RichTextTitle = title + return s +} + +// WithSubtitle sets the subtitle for the ContainerBlock. +func (s *ContainerBlock) WithSubtitle(subtitle *TextBlockObject) *ContainerBlock { + s.Subtitle = subtitle + return s +} + +// WithIcon sets the icon displayed beside the title for the ContainerBlock. +func (s *ContainerBlock) WithIcon(icon *ImageBlockElement) *ContainerBlock { + s.Icon = icon + return s +} + +// WithWidth sets the rendered width of the ContainerBlock. +func (s *ContainerBlock) WithWidth(width ContainerWidth) *ContainerBlock { + s.Width = width + return s +} + +// WithCollapsible marks the ContainerBlock collapsible and controls whether it +// starts collapsed. +func (s *ContainerBlock) WithCollapsible(collapsible, defaultCollapsed bool) *ContainerBlock { + s.IsCollapsible = collapsible + s.DefaultCollapsed = defaultCollapsed + return s +} + +// WithHeaderDivider draws a border below the container header. +func (s *ContainerBlock) WithHeaderDivider(hasHeaderDivider bool) *ContainerBlock { + s.HasHeaderDivider = hasHeaderDivider + return s +} + +// AddChildBlock appends a block to the container's child blocks. +func (s *ContainerBlock) AddChildBlock(block Block) *ContainerBlock { + s.ChildBlocks.BlockSet = append(s.ChildBlocks.BlockSet, block) + return s +} diff --git a/backend/vendor/github.com/slack-go/slack/block_conv.go b/backend/vendor/github.com/slack-go/slack/block_conv.go index 1cce78e9..2a298522 100644 --- a/backend/vendor/github.com/slack-go/slack/block_conv.go +++ b/backend/vendor/github.com/slack-go/slack/block_conv.go @@ -93,6 +93,8 @@ func (b *Blocks) UnmarshalJSON(data []byte) error { block = &CardBlock{} case "carousel": block = &CarouselBlock{} + case "container": + block = &ContainerBlock{} default: b := &UnknownBlock{raw: r} if err = json.Unmarshal(r, b); err != nil { diff --git a/backend/vendor/github.com/slack-go/slack/chat.go b/backend/vendor/github.com/slack-go/slack/chat.go index ba789c92..7fbcc191 100644 --- a/backend/vendor/github.com/slack-go/slack/chat.go +++ b/backend/vendor/github.com/slack-go/slack/chat.go @@ -29,11 +29,12 @@ const ( ) type chatResponseFull struct { - Channel string `json:"channel"` - Timestamp string `json:"ts"` // Regular message timestamp - MessageTimeStamp string `json:"message_ts"` // Ephemeral message timestamp - ScheduledMessageID string `json:"scheduled_message_id,omitempty"` // Scheduled message id - Text string `json:"text"` + Channel string `json:"channel"` + Timestamp string `json:"ts"` // Regular message timestamp + MessageTimeStamp string `json:"message_ts"` // Ephemeral message timestamp + ScheduledMessageID string `json:"scheduled_message_id,omitempty"` // Scheduled message id + Text string `json:"text"` + Message Message `json:"message"` // Full message object, as returned by chat.postMessage and chat.update SlackResponse } @@ -148,6 +149,33 @@ func (api *Client) PostMessageContext(ctx context.Context, channelID string, opt return respChannel, respTimestamp, err } +// PostMessageWithResponse sends a message to a channel and returns the full +// message object from the Slack API response. +// For more details, see PostMessageWithResponseContext documentation. +func (api *Client) PostMessageWithResponse(channelID string, options ...MsgOption) (string, string, Message, error) { + return api.PostMessageWithResponseContext(context.Background(), channelID, options...) +} + +// PostMessageWithResponseContext sends a message to a channel with a custom +// context and returns the full message object from the Slack API response. +// Unlike PostMessageContext, it exposes fields that are only available in the +// response's message object, such as Message.ThreadTimestamp, which can be +// used to detect that a threaded reply was posted un-threaded because its +// parent message was deleted. +// Slack API docs: https://api.slack.com/methods/chat.postMessage +func (api *Client) PostMessageWithResponseContext(ctx context.Context, channelID string, options ...MsgOption) (string, string, Message, error) { + response, err := api.sendResponseFull( + ctx, + channelID, + MsgOptionPost(), + MsgOptionCompose(options...), + ) + if response == nil { + return "", "", Message{}, err + } + return response.Channel, response.getMessageTimestamp(), response.Message, err +} + // PostEphemeral sends an ephemeral message to a user in a channel. // Message is escaped by default according to https://api.slack.com/docs/formatting // Use http://davestevens.github.io/slack-message-builder/ to help crafting your message. @@ -245,34 +273,47 @@ func (api *Client) SendMessage(channel string, options ...MsgOption) (string, st // SendMessageContext more flexible method for configuring messages with a custom context. // Slack API docs: https://api.slack.com/methods/chat.postMessage func (api *Client) SendMessageContext(ctx context.Context, channelID string, options ...MsgOption) (_channel string, _timestampOrScheduledMessageID string, _text string, err error) { + response, err := api.sendResponseFull(ctx, channelID, options...) + if response == nil { + return "", "", "", err + } + + if response.ScheduledMessageID != "" { + return response.Channel, response.ScheduledMessageID, response.Text, err + } else { + return response.Channel, response.getMessageTimestamp(), response.Text, err + } +} + +// sendResponseFull sends a message and returns the full response. +// It returns a nil response if the request could not be built or sent; +// otherwise the returned error is the response's error, if any. +func (api *Client) sendResponseFull(ctx context.Context, channelID string, options ...MsgOption) (*chatResponseFull, error) { var ( req *http.Request parser func(*chatResponseFull) responseParser response chatResponseFull + err error ) if req, parser, err = buildSender(api.endpoint, options...).BuildRequestContext(ctx, api.token, channelID); err != nil { - return "", "", "", err + return nil, err } if api.Debug() { reqBody, err := io.ReadAll(req.Body) if err != nil { - return "", "", "", err + return nil, err } req.Body = io.NopCloser(bytes.NewBuffer(reqBody)) api.Debugf("Sending request: %s", redactToken(reqBody)) } if _, err = doPost(api.httpclient, req, parser(&response), api); err != nil { - return "", "", "", err + return nil, err } - if response.ScheduledMessageID != "" { - return response.Channel, response.ScheduledMessageID, response.Text, response.Err() - } else { - return response.Channel, response.getMessageTimestamp(), response.Text, response.Err() - } + return &response, response.Err() } func redactToken(b []byte) []byte { diff --git a/backend/vendor/github.com/slack-go/slack/files.go b/backend/vendor/github.com/slack-go/slack/files.go index a88c9c7d..155beba2 100644 --- a/backend/vendor/github.com/slack-go/slack/files.go +++ b/backend/vendor/github.com/slack-go/slack/files.go @@ -162,6 +162,7 @@ type UploadFileParameters struct { InitialComment string Blocks Blocks Channel string + Channels []string ThreadTimestamp string AltTxt string SnippetType string @@ -197,6 +198,7 @@ type CompleteUploadExternalParameters struct { Files []FileSummary Blocks Blocks Channel string + Channels []string InitialComment string ThreadTimestamp string } @@ -489,7 +491,7 @@ func (api *Client) UploadToURL(ctx context.Context, params UploadToURLParameters return err } -// CompleteUploadExternalContext once files are uploaded, this completes the upload and shares it to the specified channel +// CompleteUploadExternalContext once files are uploaded, this completes the upload and shares it to the specified channels // Slack API docs: https://api.slack.com/methods/files.completeUploadExternal func (api *Client) CompleteUploadExternalContext(ctx context.Context, params CompleteUploadExternalParameters) (file *CompleteUploadExternalResponse, err error) { filesBytes, err := json.Marshal(params.Files) @@ -505,6 +507,9 @@ func (api *Client) CompleteUploadExternalContext(ctx context.Context, params Com if params.Channel != "" { values.Add("channel_id", params.Channel) } + if len(params.Channels) > 0 { + values.Add("channels", strings.Join(params.Channels, ",")) + } if params.InitialComment != "" { values.Add("initial_comment", params.InitialComment) } @@ -538,7 +543,7 @@ func (api *Client) UploadFile(params UploadFileParameters) (*FileSummary, error) // UploadFileContext uploads file to a given slack channel using 3 steps - // 1. Get an upload URL using files.getUploadURLExternal API // 2. Send the file as a post to the URL provided by slack -// 3. Complete the upload and share it to the specified channel using files.completeUploadExternal +// 3. Complete the upload and share it to the specified channels using files.completeUploadExternal // // Slack Docs: https://api.slack.com/messaging/files#uploading_files func (api *Client) UploadFileContext(ctx context.Context, params UploadFileParameters) (file *FileSummary, err error) { @@ -576,6 +581,7 @@ func (api *Client) UploadFileContext(ctx context.Context, params UploadFileParam Title: params.Title, }}, Channel: params.Channel, + Channels: params.Channels, InitialComment: params.InitialComment, ThreadTimestamp: params.ThreadTimestamp, Blocks: params.Blocks, diff --git a/backend/vendor/github.com/slack-go/slack/websocket_managed_conn.go b/backend/vendor/github.com/slack-go/slack/websocket_managed_conn.go index d2539aac..f58742b0 100644 --- a/backend/vendor/github.com/slack-go/slack/websocket_managed_conn.go +++ b/backend/vendor/github.com/slack-go/slack/websocket_managed_conn.go @@ -412,7 +412,7 @@ func (rtm *RTM) receiveIncomingEvent(events chan json.RawMessage) error { select { case events <- event: case <-rtm.disconnected: - rtm.Debugln("disonnected while attempting to send raw event") + rtm.Debugln("disconnected while attempting to send raw event") } } diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index 615ea1f5..4a9f7e3f 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -34,7 +34,7 @@ github.com/didip/tollbooth/v8/errors github.com/didip/tollbooth/v8/internal/time/rate github.com/didip/tollbooth/v8/libstring github.com/didip/tollbooth/v8/limiter -# github.com/dlclark/regexp2/v2 v2.2.2 +# github.com/dlclark/regexp2/v2 v2.7.1 ## explicit; go 1.25 github.com/dlclark/regexp2/v2 github.com/dlclark/regexp2/v2/helpers @@ -56,13 +56,13 @@ github.com/go-pkgz/auth/v2/token # github.com/go-pkgz/email v0.8.0 ## explicit; go 1.19 github.com/go-pkgz/email -# github.com/go-pkgz/expirable-cache/v3 v3.1.0 -## explicit; go 1.20 +# github.com/go-pkgz/expirable-cache/v3 v3.1.1 +## explicit; go 1.23 github.com/go-pkgz/expirable-cache/v3 -# github.com/go-pkgz/jrpc v0.4.0 +# github.com/go-pkgz/jrpc v0.4.2 ## explicit; go 1.24.0 github.com/go-pkgz/jrpc -# github.com/go-pkgz/lcw/v2 v2.0.0 +# github.com/go-pkgz/lcw/v2 v2.1.0 ## explicit; go 1.21 github.com/go-pkgz/lcw/v2 github.com/go-pkgz/lcw/v2/eventbus @@ -80,11 +80,11 @@ github.com/go-pkgz/repeater/v2 github.com/go-pkgz/rest github.com/go-pkgz/rest/logger github.com/go-pkgz/rest/realip -# github.com/go-pkgz/routegroup v1.6.0 +# github.com/go-pkgz/routegroup v1.6.1 ## explicit; go 1.23 github.com/go-pkgz/routegroup -# github.com/go-pkgz/syncs v1.3.2 -## explicit; go 1.20 +# github.com/go-pkgz/syncs v1.3.3 +## explicit; go 1.22 github.com/go-pkgz/syncs # github.com/golang-jwt/jwt/v5 v5.3.1 ## explicit; go 1.21 @@ -104,12 +104,6 @@ github.com/gorilla/feeds # github.com/gorilla/websocket v1.5.3 ## explicit; go 1.12 github.com/gorilla/websocket -# github.com/hashicorp/errwrap v1.1.0 -## explicit -github.com/hashicorp/errwrap -# github.com/hashicorp/go-multierror v1.1.1 -## explicit; go 1.13 -github.com/hashicorp/go-multierror # github.com/hashicorp/golang-lru/v2 v2.0.7 ## explicit; go 1.18 github.com/hashicorp/golang-lru/v2 @@ -129,8 +123,8 @@ github.com/klauspost/compress/internal/le github.com/klauspost/compress/internal/snapref github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash -# github.com/kyokomi/emoji/v2 v2.2.13 -## explicit; go 1.14 +# github.com/kyokomi/emoji/v2 v2.2.14 +## explicit; go 1.21 github.com/kyokomi/emoji/v2 # github.com/microcosm-cc/bluemonday v1.0.27 ## explicit; go 1.19 @@ -139,7 +133,7 @@ github.com/microcosm-cc/bluemonday/css # github.com/montanaflynn/stats v0.12.4 ## explicit; go 1.13 github.com/montanaflynn/stats -# github.com/redis/go-redis/v9 v9.21.0 +# github.com/redis/go-redis/v9 v9.22.0 ## explicit; go 1.24 github.com/redis/go-redis/v9 github.com/redis/go-redis/v9/auth @@ -170,7 +164,7 @@ github.com/russross/blackfriday/v2 github.com/skip2/go-qrcode github.com/skip2/go-qrcode/bitset github.com/skip2/go-qrcode/reedsolomon -# github.com/slack-go/slack v0.27.0 +# github.com/slack-go/slack v0.29.0 ## explicit; go 1.25 github.com/slack-go/slack github.com/slack-go/slack/internal/backoff @@ -278,7 +272,7 @@ golang.org/x/image/riff golang.org/x/image/vp8 golang.org/x/image/vp8l golang.org/x/image/webp -# golang.org/x/net v0.57.0 +# golang.org/x/net v0.58.0 ## explicit; go 1.25.0 golang.org/x/net/html golang.org/x/net/html/atom