Bump backend Go modules to latest
Updates every backend dependency with a newer release available, and tidies the example module alongside as any change to backend/go.mod requires.
This commit is contained in:
Dmitry Verkhoturov
committed by
Umputun
Umputun
parent
3f5b3cdd98
commit
09110c792f
@@ -3,7 +3,7 @@ module github.com/umputun/remark42/memory_store
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/go-pkgz/jrpc v0.4.0
|
||||
github.com/go-pkgz/jrpc v0.4.2
|
||||
github.com/go-pkgz/lgr v0.12.4
|
||||
github.com/jessevdk/go-flags v1.6.1
|
||||
github.com/stretchr/testify v1.12.0
|
||||
@@ -16,9 +16,9 @@ require (
|
||||
github.com/alecthomas/chroma/v2 v2.27.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 // indirect
|
||||
github.com/go-pkgz/rest v1.24.0 // indirect
|
||||
github.com/go-pkgz/routegroup v1.6.0 // indirect
|
||||
github.com/go-pkgz/routegroup v1.6.1 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
|
||||
@@ -28,7 +28,7 @@ require (
|
||||
go.etcd.io/bbolt v1.5.0 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/image v0.45.0 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -13,16 +13,16 @@ github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-pkgz/jrpc v0.4.0 h1:oD7xiGrzDkndkuCjeHGugQXxbggLSV7O1QmHhoc5pYY=
|
||||
github.com/go-pkgz/jrpc v0.4.0/go.mod h1:JFoY3bRjRyx4M3CbEVDFQStMB1m2gmQ7OjqFK7q3kOo=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/go-pkgz/jrpc v0.4.2 h1:gY5mmxp9/dFd1WsHybVZILQpF11YNWWS3Ga+Pc5aIAU=
|
||||
github.com/go-pkgz/jrpc v0.4.2/go.mod h1:ZtnMpIXYmwXh6W44XO2lE5Lh5J+6KeeMIvw+vF9xXRQ=
|
||||
github.com/go-pkgz/lgr v0.12.4 h1:lDeQ4BR28ldXrKau6BOjq7A8nHzcXz+MF4xUfV4l1Ok=
|
||||
github.com/go-pkgz/lgr v0.12.4/go.mod h1:Lw6DkNRnCPyX07mqkiUK/p+eA1opq4GKkWfWia64RA8=
|
||||
github.com/go-pkgz/rest v1.24.0 h1:GAUCgx7U8xCOC2OynLjhCRMhtnMQH4d1mTdKpQyX2yI=
|
||||
github.com/go-pkgz/rest v1.24.0/go.mod h1:dl3EWiuFB4hRTo2Sknj6UrQGFRAYvANK6/NyW8qQPxc=
|
||||
github.com/go-pkgz/routegroup v1.6.0 h1:44XHZgF6JIIldRlv+zjg6SygULASmjifnfIQjwCT0e4=
|
||||
github.com/go-pkgz/routegroup v1.6.0/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/go-pkgz/routegroup v1.6.1 h1:6I/0LabazpZsHAI+jYPeyH/KU2cvZF0bFylUScMNi+Q=
|
||||
github.com/go-pkgz/routegroup v1.6.1/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
|
||||
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
@@ -49,8 +49,8 @@ golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
|
||||
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
|
||||
+10
-12
@@ -8,19 +8,19 @@ require (
|
||||
github.com/alecthomas/chroma/v2 v2.27.0
|
||||
github.com/didip/tollbooth/v8 v8.0.1
|
||||
github.com/go-pkgz/auth/v2 v2.2.0
|
||||
github.com/go-pkgz/jrpc v0.4.0
|
||||
github.com/go-pkgz/lcw/v2 v2.0.0
|
||||
github.com/go-pkgz/jrpc v0.4.2
|
||||
github.com/go-pkgz/lcw/v2 v2.1.0
|
||||
github.com/go-pkgz/lgr v0.12.4
|
||||
github.com/go-pkgz/notify v1.4.0
|
||||
github.com/go-pkgz/repeater/v2 v2.2.0
|
||||
github.com/go-pkgz/rest v1.24.0
|
||||
github.com/go-pkgz/routegroup v1.6.0
|
||||
github.com/go-pkgz/syncs v1.3.2
|
||||
github.com/go-pkgz/routegroup v1.6.1
|
||||
github.com/go-pkgz/syncs v1.3.3
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/feeds v1.2.0
|
||||
github.com/jessevdk/go-flags v1.6.1
|
||||
github.com/kyokomi/emoji/v2 v2.2.13
|
||||
github.com/kyokomi/emoji/v2 v2.2.14
|
||||
github.com/microcosm-cc/bluemonday v1.0.27
|
||||
github.com/rs/xid v1.6.0
|
||||
github.com/russross/blackfriday/v2 v2.1.0
|
||||
@@ -30,7 +30,7 @@ require (
|
||||
go.uber.org/goleak v1.3.0
|
||||
golang.org/x/crypto v0.55.0
|
||||
golang.org/x/image v0.45.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
)
|
||||
|
||||
@@ -40,21 +40,19 @@ require (
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dghubble/oauth1 v0.7.3 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 // indirect
|
||||
github.com/go-oauth2/oauth2/v4 v4.5.4 // indirect
|
||||
github.com/go-pkgz/email v0.8.0 // indirect
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.0 // indirect
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.1 // indirect
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/montanaflynn/stats v0.12.4 // indirect
|
||||
github.com/redis/go-redis/v9 v9.21.0 // indirect
|
||||
github.com/redis/go-redis/v9 v9.22.0 // indirect
|
||||
github.com/rrivera/identicon v0.0.0-20240116195454-d5ba35832c0d // indirect
|
||||
github.com/slack-go/slack v0.27.0 // indirect
|
||||
github.com/slack-go/slack v0.29.0 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.2.0 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
|
||||
+22
-29
@@ -12,10 +12,8 @@ github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r
|
||||
github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8=
|
||||
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
|
||||
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
|
||||
github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y=
|
||||
github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg=
|
||||
github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw=
|
||||
github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
|
||||
@@ -34,8 +32,8 @@ github.com/dghubble/oauth1 v0.7.3 h1:EkEM/zMDMp3zOsX2DC/ZQ2vnEX3ELK0/l9kb+vs4ptE
|
||||
github.com/dghubble/oauth1 v0.7.3/go.mod h1:oxTe+az9NSMIucDPDCCtzJGsPhciJV33xocHfcR2sVY=
|
||||
github.com/didip/tollbooth/v8 v8.0.1 h1:VAAapTo1t4Bn6bbpcHjuovwoa9u3JH++wgjbpWv+rB8=
|
||||
github.com/didip/tollbooth/v8 v8.0.1/go.mod h1:oEd9l+ep373d7DmvKLc0a5gasPOev2mTewi6KPQBGJ4=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4=
|
||||
github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/gavv/httpexpect v2.0.0+incompatible h1:1X9kcRshkSKEjNJJxX9Y9mQ5BRfbxU5kORdjhlA1yX8=
|
||||
@@ -46,12 +44,12 @@ github.com/go-pkgz/auth/v2 v2.2.0 h1:vQO+GTFDjAaBNSdcLLLr3Xibka67GZPtkRRItVVS4ow
|
||||
github.com/go-pkgz/auth/v2 v2.2.0/go.mod h1:iZx2JiGZ8Aef+wM0BPLMQY8aur4fLE0uyPhFRO9dYQ4=
|
||||
github.com/go-pkgz/email v0.8.0 h1:6+Tgjfj7zFccFCPmURV2spKDXDb7aX/iWXBY2hBa9ww=
|
||||
github.com/go-pkgz/email v0.8.0/go.mod h1:+wgi4x7S33IuCzfcCM5euN0GwQG6XvO/PBLxrNffYLI=
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.0 h1:s05P851/O6QJ6Mc+7o2bh9aGtD3romB1SxDTXifdoqc=
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.0/go.mod h1:6pVgNleydKPj0J2/mzrI02/RDo4ivKx5v2XlNmIjhjo=
|
||||
github.com/go-pkgz/jrpc v0.4.0 h1:oD7xiGrzDkndkuCjeHGugQXxbggLSV7O1QmHhoc5pYY=
|
||||
github.com/go-pkgz/jrpc v0.4.0/go.mod h1:JFoY3bRjRyx4M3CbEVDFQStMB1m2gmQ7OjqFK7q3kOo=
|
||||
github.com/go-pkgz/lcw/v2 v2.0.0 h1:gTwXpiJBhQeA1rXuqkRuLcV79uATFna8CckH8ZBBrH0=
|
||||
github.com/go-pkgz/lcw/v2 v2.0.0/go.mod h1:yxJHOn+IbQBQHxUqkCtMrbGjIfdYcsBAZcVCBaL1Va8=
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.1 h1:ryHiSI5gBE8aJ0Jt90VFMKZxe/cosf2dZPDcUTMaHNg=
|
||||
github.com/go-pkgz/expirable-cache/v3 v3.1.1/go.mod h1:peJAuIDjP76Uuc9NK55ljQlBtwwmJDvx4CnMyUcsP40=
|
||||
github.com/go-pkgz/jrpc v0.4.2 h1:gY5mmxp9/dFd1WsHybVZILQpF11YNWWS3Ga+Pc5aIAU=
|
||||
github.com/go-pkgz/jrpc v0.4.2/go.mod h1:ZtnMpIXYmwXh6W44XO2lE5Lh5J+6KeeMIvw+vF9xXRQ=
|
||||
github.com/go-pkgz/lcw/v2 v2.1.0 h1:JAGUHRQPon658XimxIUwaqPgOPfIKa850qO8jpFJchc=
|
||||
github.com/go-pkgz/lcw/v2 v2.1.0/go.mod h1:UUo4cgD6oTPooBuUslVaWqOYZAGnh/91SoaB5DBWC8s=
|
||||
github.com/go-pkgz/lgr v0.12.4 h1:lDeQ4BR28ldXrKau6BOjq7A8nHzcXz+MF4xUfV4l1Ok=
|
||||
github.com/go-pkgz/lgr v0.12.4/go.mod h1:Lw6DkNRnCPyX07mqkiUK/p+eA1opq4GKkWfWia64RA8=
|
||||
github.com/go-pkgz/notify v1.4.0 h1:4pP7UGdYqFO7e7V3OsQStYF006CO0cCh1ahdawt6l18=
|
||||
@@ -60,10 +58,10 @@ github.com/go-pkgz/repeater/v2 v2.2.0 h1:8nZR/NaknmLfx2YMHbr78u9OL4Xj+8+romm9dz4
|
||||
github.com/go-pkgz/repeater/v2 v2.2.0/go.mod h1:RgX5vUbLKq7PV82QUDP5pFbQS1os4Z+U9XzKymK23A8=
|
||||
github.com/go-pkgz/rest v1.24.0 h1:GAUCgx7U8xCOC2OynLjhCRMhtnMQH4d1mTdKpQyX2yI=
|
||||
github.com/go-pkgz/rest v1.24.0/go.mod h1:dl3EWiuFB4hRTo2Sknj6UrQGFRAYvANK6/NyW8qQPxc=
|
||||
github.com/go-pkgz/routegroup v1.6.0 h1:44XHZgF6JIIldRlv+zjg6SygULASmjifnfIQjwCT0e4=
|
||||
github.com/go-pkgz/routegroup v1.6.0/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/go-pkgz/syncs v1.3.2 h1:gmioASlJNy3gNosPlgvWOM2QP0Hdjzn2u+/sUShgd8E=
|
||||
github.com/go-pkgz/syncs v1.3.2/go.mod h1:qjgzpp7OpuhDf7BWsW/FHCu9DLjE32NPy6/vXAXT/Cw=
|
||||
github.com/go-pkgz/routegroup v1.6.1 h1:6I/0LabazpZsHAI+jYPeyH/KU2cvZF0bFylUScMNi+Q=
|
||||
github.com/go-pkgz/routegroup v1.6.1/go.mod h1:Pmu04fhgWhRtBMIJ8HXppnnzOPjnL/IEPBIdO2zmeqg=
|
||||
github.com/go-pkgz/syncs v1.3.3 h1:fFRK+eqCIFddxEiDi6ob5oQh3/NuNkjuwWQJzZ0b9lU=
|
||||
github.com/go-pkgz/syncs v1.3.3/go.mod h1:lAp+w+qbRm6+pwINcfc9BVK+4F4NEoERYditsF1uENA=
|
||||
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
|
||||
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
@@ -84,11 +82,6 @@ github.com/gorilla/feeds v1.2.0 h1:O6pBiXJ5JHhPvqy53NsjKOThq+dNFm8+DFrxBEdzSCc=
|
||||
github.com/gorilla/feeds v1.2.0/go.mod h1:WMib8uJP3BbY+X8Szd1rA5Pzhdfh+HCCAYT2z7Fza6Y=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
@@ -107,16 +100,16 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO7U=
|
||||
github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
|
||||
github.com/kyokomi/emoji/v2 v2.2.14 h1:YOF6VL52613M0Qr9v4puJDD9QQPmyyjXedDDlrGzH80=
|
||||
github.com/kyokomi/emoji/v2 v2.2.14/go.mod h1:1AnYl9IgmJZXKd5m1PEijyyUw85SqYsuAr8lpU/s+9s=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
|
||||
github.com/montanaflynn/stats v0.12.4 h1:amtNRsti20yIhcrkfUJGwoYqBR82jKQFE8SNNYVgGn0=
|
||||
github.com/montanaflynn/stats v0.12.4/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
|
||||
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
|
||||
github.com/redis/go-redis/v9 v9.21.0 h1:FPBE4hhbAke+TLmcY3WkpbDffJEomdqPn3HYiqAtL9E=
|
||||
github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||
github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0=
|
||||
github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rrivera/identicon v0.0.0-20240116195454-d5ba35832c0d h1:l3+2LWCbVxn5itfvXAfH9n4YL9jh8l1g5zcncbIc1cs=
|
||||
@@ -129,8 +122,8 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
|
||||
github.com/slack-go/slack v0.27.0 h1:VWOpUzOK6UAPCCQlFxl79jhv8a/b+GOSJMnWziDJ8B8=
|
||||
github.com/slack-go/slack v0.27.0/go.mod h1:UEe+jmo9WLlwHB04qsOrTDvqM7Aa4rQL3O5wF3n0hx4=
|
||||
github.com/slack-go/slack v0.29.0 h1:ohhMNgp9DmPKiLhH/pNZV4NxhOXKgNy0SH8FzVHNerI=
|
||||
github.com/slack-go/slack v0.29.0/go.mod h1:UEe+jmo9WLlwHB04qsOrTDvqM7Aa4rQL3O5wF3n0hx4=
|
||||
github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
|
||||
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
@@ -200,8 +193,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
||||
+43
-5
@@ -38,7 +38,7 @@ if isMatch, _ := re.MatchString(`Something to match`); isMatch {
|
||||
}
|
||||
```
|
||||
|
||||
The only error that the `*Match*` methods *should* return is a Timeout if you set the `re.MatchTimeout` field. Any other error is a bug in the `regexp2` package. If you need more details about capture groups in a match then use the `FindStringMatch` method, like so:
|
||||
The `*Match*` methods can return a timeout error if you set the `re.MatchTimeout` field, or `ErrBacktrackingStackLimit` if a match exceeds its configured backtracking stack size. Any other error is a bug in the `regexp2` package. If you need more details about capture groups in a match then use the `FindStringMatch` method, like so:
|
||||
|
||||
```go
|
||||
if m, _ := re.FindStringMatch(`Something to match`); m != nil {
|
||||
@@ -92,6 +92,17 @@ notEmoji := regexp2.MustCompile(`\P{Emoji}+`)
|
||||
|
||||
Valid property names and aliases come from Unicode 17.0.0 [`PropertyAliases.txt`](https://www.unicode.org/Public/17.0.0/ucd/PropertyAliases.txt). Valid property values and aliases come from Unicode 17.0.0 [`PropertyValueAliases.txt`](https://www.unicode.org/Public/17.0.0/ucd/PropertyValueAliases.txt). The generated tables use Unicode 17.0.0 data from [`DerivedCoreProperties.txt`](https://www.unicode.org/Public/17.0.0/ucd/DerivedCoreProperties.txt), [`emoji/emoji-data.txt`](https://www.unicode.org/Public/17.0.0/ucd/emoji/emoji-data.txt), [`auxiliary/GraphemeBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/GraphemeBreakProperty.txt), [`auxiliary/WordBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/WordBreakProperty.txt), and [`auxiliary/SentenceBreakProperty.txt`](https://www.unicode.org/Public/17.0.0/ucd/auxiliary/SentenceBreakProperty.txt) for the package-local properties whose data changes more frequently than the Go standard library tables.
|
||||
|
||||
## Additional Perl and PCRE syntax
|
||||
|
||||
The default mode supports the following syntax:
|
||||
|
||||
* `\Q...\E` quotes every character between `\Q` and `\E`. If `\E` is omitted, quoting continues to the end of the pattern. This also works inside character classes.
|
||||
* `\R` matches one Unicode newline sequence: CRLF as a single sequence, or LF, VT, FF, CR, NEL, line separator, or paragraph separator.
|
||||
* `\X` atomically matches one Unicode 17.0.0 extended grapheme cluster. This includes combining sequences, Hangul syllables, regional-indicator pairs, emoji ZWJ sequences, and Indic conjuncts.
|
||||
* A `+` after a quantifier makes it possessive: `*+`, `++`, `?+`, and `{m,n}+`. For example, `a*+` is equivalent to `(?>a*)` and will not give characters back when the remainder of the pattern fails.
|
||||
|
||||
`RE2` mode also supports `\Q...\E` and possessive quantifiers, but keeps `\R` and `\X` as literal `R` and `X` identity escapes. `ECMAScript` mode does not enable any of this syntax: `\Q`, `\E`, `\R`, and `\X` retain ECMAScript identity-escape behavior, and possessive quantifiers remain invalid.
|
||||
|
||||
## `regexp` compatibility adapter
|
||||
|
||||
The `github.com/dlclark/regexp2/v2/compat` package provides an adapter for callers that want the same `Find*` and `Match*` method signatures as the standard library's `regexp.Regexp`, while still using the `regexp2` engine.
|
||||
@@ -128,7 +139,7 @@ func findWords(re compat.Matcher, input string) []string {
|
||||
}
|
||||
```
|
||||
|
||||
Because those standard-library method signatures do not return errors, the adapter panics if the wrapped regexp2 matcher returns an error such as a match timeout. Use the main `regexp2` APIs directly when you need to handle timeouts as errors.
|
||||
Because those standard-library method signatures do not return errors, the adapter panics if the wrapped regexp2 matcher returns an error such as a match timeout or `ErrBacktrackingStackLimit`. Use the main `regexp2` APIs directly when you need to handle match errors directly.
|
||||
|
||||
## Compile options
|
||||
|
||||
@@ -146,6 +157,7 @@ Performance tuning options override the default cache settings:
|
||||
```go
|
||||
re := regexp2.MustCompile(`Your pattern`,
|
||||
regexp2.IgnoreCase,
|
||||
regexp2.OptionMaxBacktrackingStackSize(200000),
|
||||
regexp2.OptionMaxCachedRuneBufferLength(64*1024),
|
||||
regexp2.OptionMaxCachedReplacerDataEntries(8),
|
||||
)
|
||||
@@ -164,23 +176,30 @@ The defaults are intentionally bounded:
|
||||
| `OptionMaintainCaptureOrder()` | false | Parser capture-slot assignment for mixed named and unnamed captures. | None at match time. This changes compile-time capture numbering only. | Keeps named and unnamed captures in pattern order instead of appending named captures after unnamed captures. This can change numeric backreference meaning, so it is caller-controlled rather than an inline regex option. |
|
||||
| `OptionDebug()` | false | Compile dumps and runner tracing. | Debug output volume only. | Useful for diagnostics, but it can produce noisy output and slower traced matching. |
|
||||
| `OptionIsCodeGen()` | false | Compile-time find-optimization analysis for [`regexp2cg`](https://github.com/dlclark/regexp2cg). | Per compiled regexp, during `Compile` or `MustCompile`. | Enables more expensive analysis intended for generated engines. Do not use it for normal interpreter execution; the interpreter defaults intentionally avoid this extra compile-time cost. |
|
||||
| `OptionMaxBacktrackingStackSize(n)` | 100,000 | The interpreter's per-match backtracking stack. | Per pooled runner. The initial allocation and subsequent growth are capped at the configured number of integer slots; the runner pool may retain stacks at their high-water size for reuse. | Lowering this bounds backtracking memory more tightly but may reject complex matches sooner with `ErrBacktrackingStackLimit`. Raising it permits deeper backtracking and increases possible memory use. A negative value disables the limit. |
|
||||
| `OptionMaxCachedRuneBufferLength(n)` | 256K runes | String APIs that run through pooled runners, such as `MatchString` and replacement-pattern `Replace`, when converting input strings to the engine's internal `[]rune` representation. | Process-wide shared `sync.Pool` retention by size class. This does not grow per compiled regexp or per input string; the practical working set follows recent and concurrent use across all regexps and can be dropped by GC. | Raising this lets calls use larger pooled rune buffers and can reduce allocations for repeated matches against large strings. Lowering it prevents larger buffers from being borrowed or returned, so large inputs allocate directly. |
|
||||
| `OptionMaxCachedReplaceBufferLength(n)` | 256 KB | Replacement-pattern `Replace` calls that build output through a shared byte buffer. | Process-wide shared `sync.Pool` retention by size class after replacement-pattern `Replace` runs. It does not grow from evaluator-based `ReplaceFunc` output and is shared across compiled regexps. | Raising this lets larger replacement outputs use pooled buffers and can reduce allocations. Lowering it prevents larger output buffers from being retained, so large replacements allocate directly. |
|
||||
| `OptionMaxCachedReplacerDataEntries(n)` | `16` | `Replace` with replacement pattern strings, after the replacement pattern is parsed into reusable replacement data. | Per compiled regexp. The cache grows as distinct cacheable replacement strings are used with `Replace`, up to this entry count. | Raising this helps when a single compiled regexp is used with many recurring replacement patterns. It increases per-regexp cache memory and lock-protected cache bookkeeping. Setting it to `0` disables this cache. |
|
||||
| `OptionMaxCachedReplacerDataBytes(n)` | 4 KB | The parsed replacement-pattern cache. Replacement strings longer than this are parsed for the call but not retained. | Per compiled regexp, combined with `OptionMaxCachedReplacerDataEntries`. Only replacement strings whose source text is at or below this size can add parsed data to the cache. | Raising this helps if large replacement patterns are reused. It can retain more memory per cached replacement. Lowering it avoids keeping unusual large replacement patterns around. |
|
||||
| `OptionDisableCharClassASCIIBitmap()` | false | Compile-time preparation of character classes and first-character prefix sets. By default, character classes with ASCII membership get a small bitmap used by `CharIn`. | Per compiled regexp, during `Compile` or `MustCompile`. Each eligible character class can hold one small bitmap; this does not scale with match concurrency or input size. | Leaving this false speeds up ASCII-heavy character class checks at the cost of a small amount of per-char-class memory and compile-time work. Setting to true can reduce memory for large numbers of compiled char classes in regexps, but ASCII character class matching may be slower. |
|
||||
|
||||
For `OptionMaxBacktrackingStackSize`, set `n` to a negative value to allow unbounded stack growth. Setting it to `0` permits no backtracking stack entries, so most interpreted matches will return `ErrBacktrackingStackLimit`.
|
||||
|
||||
For pooled buffer cache options, set `n` to `0` to disable pooling, or `-1` to allow all built-in size classes. The rune buffer classes are 1K, 4K, 16K, 64K, and 256K runes. The replacement byte buffer classes are 4 KB, 16 KB, 64 KB, 256 KB, and 1 MB. By default the 1 MB pool is unused. For replacement data byte-size cache options, `-1` means unbounded. For entry-count cache options, set `n` to `0` to disable the cache.
|
||||
|
||||
## Compare `regexp` and `regexp2`
|
||||
| Category | regexp | regexp2 |
|
||||
| --- | --- | --- |
|
||||
| Catastrophic backtracking possible | no, constant execution time guarantees | yes, if your pattern is at risk you can use the `re.MatchTimeout` field |
|
||||
| Catastrophic backtracking possible | no, constant execution time guarantees | yes; backtracking stack growth is bounded by default, and `re.MatchTimeout` can also bound match duration |
|
||||
| Python-style capture groups `(?P<name>re)` | yes | no (yes in RE2 compat mode) |
|
||||
| .NET-style capture groups `(?<name>re)` or `(?'name're)` | yes | yes |
|
||||
| comments `(?#comment)` | no | yes |
|
||||
| branch numbering reset `(?\|a\|b)` | no | no |
|
||||
| possessive match `(?>re)` | no | yes |
|
||||
| atomic group `(?>re)` | no | yes |
|
||||
| possessive quantifiers `*+`, `++`, `?+`, `{m,n}+` | no | yes |
|
||||
| literal quoting `\Q...\E` | yes | yes |
|
||||
| Unicode newline sequence `\R` | no | yes (default mode only) |
|
||||
| extended grapheme cluster `\X` | no | yes (default mode only) |
|
||||
| positive lookahead `(?=re)` | no | yes |
|
||||
| negative lookahead `(?!re)` | no | yes |
|
||||
| positive lookbehind `(?<=re)` | no | yes |
|
||||
@@ -199,6 +218,8 @@ The default behavior of `regexp2` is to match the .NET regexp engine, however th
|
||||
* change singleline behavior for `$` to only match end of string (like RE2) (see [#24](https://github.com/dlclark/regexp2/issues/24))
|
||||
* change the character classes `\d` `\s` and `\w` to match the same characters as RE2. NOTE: if you also use the `ECMAScript` option then this will change the `\s` character class to match ECMAScript instead of RE2. ECMAScript allows more whitespace characters in `\s` than RE2 (but still fewer than the the default behavior).
|
||||
* allow character escape sequences to have defaults. For example, by default `\_` isn't a known character escape and will fail to compile, but in RE2 mode it will match the literal character `_`
|
||||
* support RE2-style literal quoting with `\Q...\E`
|
||||
* support possessive quantifiers (`*+`, `++`, `?+`, and `{m,n}+`) as a regexp2 extension
|
||||
|
||||
```go
|
||||
re := regexp2.MustCompile(`Your RE2-compatible pattern`, regexp2.RE2)
|
||||
@@ -212,7 +233,22 @@ This feature is a work in progress and I'm open to ideas for more things to put
|
||||
## Catastrophic Backtracking and Timeouts
|
||||
|
||||
`regexp2` supports features that can lead to catastrophic backtracking.
|
||||
`Regexp.MatchTimeout` can be set to to limit the impact of such behavior; the
|
||||
Each compiled regexp limits its per-match backtracking stack to 100,000
|
||||
slots by default. If a match would exceed that limit, it stops and returns
|
||||
`ErrBacktrackingStackLimit`. Callers can identify it with
|
||||
`errors.Is(err, regexp2.ErrBacktrackingStackLimit)`. The limit can be changed at
|
||||
compile time; a negative value restores the previous unbounded behavior:
|
||||
|
||||
```go
|
||||
re := regexp2.MustCompile(pattern, regexp2.OptionMaxBacktrackingStackSize(200000))
|
||||
// regexp2.OptionMaxBacktrackingStackSize(-1) disables the limit.
|
||||
```
|
||||
|
||||
This limit bounds the interpreter's backtracking stack, not total match time or
|
||||
all memory used by a match. Literal empty expressions repeated any number of
|
||||
times are optimized away and do not consume backtracking stack space.
|
||||
|
||||
`Regexp.MatchTimeout` can be set to limit the impact of such behavior; the
|
||||
match will fail with an error after approximately MatchTimeout. No timeout
|
||||
checks are done by default.
|
||||
|
||||
@@ -276,6 +312,8 @@ This flag should not be treated as compatibility with C#'s `RegexOptions.ECMAScr
|
||||
|
||||
Additionally a Unicode mode is provided which allows parsing of `\u{CodePoint}` syntax only when both `ECMAScript` and `Unicode` are provided.
|
||||
|
||||
Perl/PCRE extensions `\Q...\E`, `\R`, `\X`, and possessive quantifiers are intentionally not enabled in this mode. The letter escapes retain the engine's existing ECMAScript identity-escape behavior.
|
||||
|
||||
## Potential bugs
|
||||
I've run a battery of tests against regexp2 from various sources and found the debug output matches the .NET engine, but .NET and Go handle strings very differently. I've attempted to handle these differences, but most of my testing deals with basic ASCII with a little bit of multi-byte Unicode. There's a chance that there are bugs in the string handling related to character sets with supplementary Unicode chars. Right-to-Left support is coded, but not well tested either.
|
||||
|
||||
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
package regexp2
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/dlclark/regexp2/v2/helpers"
|
||||
)
|
||||
|
||||
// decodedInput is the engine's rune view of a string. When the pattern cannot
|
||||
// look behind a candidate start, the slice may begin at that candidate instead
|
||||
// of byte 0.
|
||||
type decodedInput struct {
|
||||
runes []rune
|
||||
pooled *[]rune
|
||||
runeStart int // index in runes of the requested startAt; -1 if not a rune boundary
|
||||
runeOffset int // original-string rune index of runes[0]
|
||||
byteOffset int // original-string byte index of runes[0]
|
||||
}
|
||||
|
||||
// decodeFrom is the first original-string byte that must be decoded. 0 means
|
||||
// the whole string; a later index is only used when slicing is safe.
|
||||
func (re *Regexp) decodeFrom(s string, startAt int) int {
|
||||
if startAt <= 0 || re.RightToLeft() {
|
||||
return 0
|
||||
}
|
||||
need := re.decodeLeftContextRunes()
|
||||
if need < 0 {
|
||||
return 0
|
||||
}
|
||||
if need > 0 {
|
||||
return prevRuneByte(s, startAt)
|
||||
}
|
||||
return startAt
|
||||
}
|
||||
|
||||
func (re *Regexp) decodeLeftContextRunes() int {
|
||||
if re.code != nil {
|
||||
return re.code.LeftContextRunes
|
||||
}
|
||||
return re.leftContextRunes
|
||||
}
|
||||
|
||||
// decodeString converts s to []rune for the MatchString startAt<=0 path.
|
||||
// Keep this as close as possible to a single UTF-8 walk so 4–30 byte
|
||||
// matches stay cheap.
|
||||
func decodeString(s string, maxCachedLength int) ([]rune, *[]rune) {
|
||||
buf, pooled := pooledRuneBuffers.get(len(s), maxCachedLength)
|
||||
if len(s) >= helpers.ASCIIScanMin {
|
||||
n, _ := helpers.DecodeString(s, buf)
|
||||
return buf[:n], pooled
|
||||
}
|
||||
n := 0
|
||||
for _, ch := range s {
|
||||
buf[n] = ch
|
||||
n++
|
||||
}
|
||||
return buf[:n], pooled
|
||||
}
|
||||
|
||||
func (re *Regexp) decodeStringInput(s string, startAt int, pooled bool) decodedInput {
|
||||
maxCachedLength := 0
|
||||
if pooled {
|
||||
maxCachedLength = re.optimizations.MaxCachedRuneBufferLength
|
||||
}
|
||||
return decodeInput(s, startAt, re.decodeFrom(s, startAt), maxCachedLength, true)
|
||||
}
|
||||
|
||||
// decodeInput converts s[decodeFrom:] to runes. startAt is a byte index in s.
|
||||
// If startAt < 0 or is not a rune boundary, runeStart is -1. needOffsets walks
|
||||
// the skipped prefix to fill runeOffset; MatchString does not need that field.
|
||||
func decodeInput(s string, startAt, decodeFrom, maxCachedLength int, needOffsets bool) decodedInput {
|
||||
if decodeFrom < 0 {
|
||||
decodeFrom = 0
|
||||
}
|
||||
if decodeFrom > len(s) {
|
||||
decodeFrom = len(s)
|
||||
}
|
||||
|
||||
substr := s[decodeFrom:]
|
||||
buf, pooledBuffer := pooledRuneBuffers.get(len(substr), maxCachedLength)
|
||||
n, ascii := helpers.DecodeString(substr, buf)
|
||||
runes := buf[:n]
|
||||
|
||||
out := decodedInput{
|
||||
runes: runes,
|
||||
pooled: pooledBuffer,
|
||||
byteOffset: decodeFrom,
|
||||
runeStart: startRuneIndex(s, startAt, decodeFrom, n, ascii),
|
||||
}
|
||||
|
||||
if !needOffsets {
|
||||
return out
|
||||
}
|
||||
|
||||
// Byte index equals rune index only when the skipped prefix is also ASCII.
|
||||
if ascii && (decodeFrom == 0 || helpers.IsASCII(s[:decodeFrom])) {
|
||||
out.runeOffset = decodeFrom
|
||||
return out
|
||||
}
|
||||
if decodeFrom > 0 {
|
||||
out.runeOffset = utf8.RuneCountInString(s[:decodeFrom])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func startRuneIndex(s string, startAt, decodeFrom, runeCount int, ascii bool) int {
|
||||
if startAt < 0 {
|
||||
return -1
|
||||
}
|
||||
if startAt < decodeFrom || startAt > len(s) {
|
||||
return -1
|
||||
}
|
||||
if startAt == decodeFrom {
|
||||
return 0
|
||||
}
|
||||
if startAt == len(s) {
|
||||
return runeCount
|
||||
}
|
||||
if ascii {
|
||||
return startAt - decodeFrom
|
||||
}
|
||||
rel := startAt - decodeFrom
|
||||
i := 0
|
||||
for strIdx := range s[decodeFrom:] {
|
||||
if strIdx == rel {
|
||||
return i
|
||||
}
|
||||
if strIdx > rel {
|
||||
return -1
|
||||
}
|
||||
i++
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func prevRuneByte(s string, byteIndex int) int {
|
||||
if byteIndex <= 0 {
|
||||
return 0
|
||||
}
|
||||
_, size := utf8.DecodeLastRuneInString(s[:byteIndex])
|
||||
if size <= 0 {
|
||||
return byteIndex - 1
|
||||
}
|
||||
return byteIndex - size
|
||||
}
|
||||
|
||||
func (d decodedInput) release() {
|
||||
if d.pooled != nil {
|
||||
*d.pooled = d.runes
|
||||
pooledRuneBuffers.put(d.pooled)
|
||||
}
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package helpers
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const asciiHighBits = 0x8080808080808080
|
||||
|
||||
// ASCIIScanMin is the input length where a separate ASCII probe plus a tight
|
||||
// copy beats a single UTF-8 range loop. Short matches stay on the one-pass path.
|
||||
const ASCIIScanMin = 64
|
||||
|
||||
// ASCIISearchMin is the haystack length where stdlib/SIMD string search beats
|
||||
// a scalar first-rune scan. Below this, needle setup and unaligned
|
||||
// bytes.Index retries dominate.
|
||||
const ASCIISearchMin = 32
|
||||
|
||||
// IsASCII reports whether s contains only bytes below utf8.RuneSelf.
|
||||
func IsASCII(s string) bool {
|
||||
n := len(s)
|
||||
if n == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
b := unsafe.Slice(unsafe.StringData(s), n)
|
||||
i := 0
|
||||
for ; i < n && uintptr(unsafe.Pointer(&b[i]))&7 != 0; i++ {
|
||||
if b[i] >= utf8.RuneSelf {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for ; i+8 <= n; i += 8 {
|
||||
v := *(*uint64)(unsafe.Pointer(&b[i]))
|
||||
if v&asciiHighBits != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for ; i < n; i++ {
|
||||
if b[i] >= utf8.RuneSelf {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// IsASCIIRunes reports whether every rune in in is an ASCII code point.
|
||||
func IsASCIIRunes(in []rune) bool {
|
||||
for _, ch := range in {
|
||||
if ch >= utf8.RuneSelf {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// DecodeString writes the runes of s into buf, which must have length >= len(s).
|
||||
// It returns the rune count and whether s was all ASCII.
|
||||
func DecodeString(s string, buf []rune) (n int, ascii bool) {
|
||||
if len(s) >= ASCIIScanMin && IsASCII(s) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
buf[i] = rune(s[i])
|
||||
}
|
||||
return len(s), true
|
||||
}
|
||||
ascii = true
|
||||
for _, ch := range s {
|
||||
if ch >= utf8.RuneSelf {
|
||||
ascii = false
|
||||
}
|
||||
buf[n] = ch
|
||||
n++
|
||||
}
|
||||
return n, ascii
|
||||
}
|
||||
+107
-49
@@ -11,11 +11,30 @@ import (
|
||||
)
|
||||
|
||||
func IndexOfAny(in []rune, find []rune) int {
|
||||
// special case
|
||||
if len(find) == 0 {
|
||||
switch len(find) {
|
||||
case 0:
|
||||
return -1
|
||||
case 1:
|
||||
return IndexOfAny1(in, find[0])
|
||||
case 2:
|
||||
return IndexOfAny2(in, find[0], find[1])
|
||||
case 3:
|
||||
return IndexOfAny3(in, find[0], find[1], find[2])
|
||||
}
|
||||
|
||||
if IsASCIIRunes(find) {
|
||||
var bits [2]uint64
|
||||
for _, c := range find {
|
||||
bits[c>>6] |= 1 << (c & 63)
|
||||
}
|
||||
for i, c := range in {
|
||||
if uint32(c) < 128 && bits[c>>6]&(1<<(c&63)) != 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
// naive version
|
||||
|
||||
for i, c := range in {
|
||||
if slices.Contains(find, c) {
|
||||
return i
|
||||
@@ -25,8 +44,10 @@ func IndexOfAny(in []rune, find []rune) int {
|
||||
}
|
||||
|
||||
func IndexOfAny1(in []rune, find rune) int {
|
||||
//TODO: bytes optimization?
|
||||
return slices.Index(in, find)
|
||||
if len(in) < ASCIISearchMin {
|
||||
return slices.Index(in, find)
|
||||
}
|
||||
return indexOfRuneBytes(runeSliceBytes(in), find)
|
||||
}
|
||||
|
||||
func IndexOfAny2(in []rune, find1, find2 rune) int {
|
||||
@@ -125,28 +146,34 @@ func IndexFunc(in []rune, f func(ch rune) bool) int {
|
||||
}
|
||||
|
||||
func IndexOfAnyExceptInSet(in []rune, set syntax.CharSet) int {
|
||||
//TODO: this
|
||||
panic("not implemented")
|
||||
for i, c := range in {
|
||||
if !set.CharIn(c) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func LastIndexOf(in []rune, find []rune) int {
|
||||
end := len(in) - len(find)
|
||||
first := find[0]
|
||||
lastOffset := len(find) - 1
|
||||
last := find[lastOffset]
|
||||
for i := end; i >= 0; i-- {
|
||||
//TODO: check 2 chars needed?
|
||||
// match start and end...check the middle
|
||||
if in[i] == first && in[i+lastOffset] == last {
|
||||
// found our first char
|
||||
// check if the rest are equal
|
||||
if bytesEqual(in[i:i+len(find)], find) {
|
||||
return i
|
||||
}
|
||||
if len(find) == 0 {
|
||||
return len(in)
|
||||
}
|
||||
if len(in) < len(find) {
|
||||
return -1
|
||||
}
|
||||
haystack := runeSliceBytes(in)
|
||||
needle := runeSliceBytes(find)
|
||||
end := len(haystack)
|
||||
for end >= len(needle) {
|
||||
idx := bytes.LastIndex(haystack[:end], needle)
|
||||
if idx < 0 {
|
||||
return -1
|
||||
}
|
||||
if idx%4 == 0 {
|
||||
return idx / 4
|
||||
}
|
||||
end = idx + len(needle) - 1
|
||||
}
|
||||
|
||||
//not found
|
||||
return -1
|
||||
}
|
||||
|
||||
@@ -160,15 +187,10 @@ func LastIndexOfAnyExcept1(in []rune, not rune) int {
|
||||
}
|
||||
|
||||
func LastIndexOfAny1(in []rune, find rune) int {
|
||||
for i := len(in) - 1; i >= 0; i-- {
|
||||
if in[i] == find {
|
||||
// found our char
|
||||
return i
|
||||
}
|
||||
if len(in) == 0 {
|
||||
return -1
|
||||
}
|
||||
|
||||
//not found
|
||||
return -1
|
||||
return lastIndexOfRuneBytes(runeSliceBytes(in), find)
|
||||
}
|
||||
|
||||
func LastIndexOfAnyInRange(in []rune, first, last rune) int {
|
||||
@@ -291,30 +313,66 @@ func foldASCII(c rune) rune {
|
||||
}
|
||||
|
||||
func IndexOf(in []rune, find []rune) int {
|
||||
/*
|
||||
Since we auto-gen the find code this shouldn't happen
|
||||
if len(find) == 0 {
|
||||
//special case
|
||||
return -1
|
||||
}*/
|
||||
if len(find) == 0 {
|
||||
return 0
|
||||
}
|
||||
if len(in) < len(find) {
|
||||
return -1
|
||||
}
|
||||
end := len(in) - len(find)
|
||||
first := find[0]
|
||||
//TODO: benchmark checking last char too or first two chars
|
||||
for i := 0; i <= end; i++ {
|
||||
// match start...check the rest
|
||||
if in[i] == first {
|
||||
// found our first char
|
||||
// check if the rest are equal
|
||||
if bytesEqual(in[i:i+len(find)], find) {
|
||||
return i
|
||||
}
|
||||
/*if slices.Equal(in[i:i+len(find)], find) {
|
||||
return i
|
||||
}*/
|
||||
for i := 0; i <= end; {
|
||||
off := IndexOfAny1(in[i:end+1], first)
|
||||
if off < 0 {
|
||||
return -1
|
||||
}
|
||||
i += off
|
||||
if bytesEqual(in[i:i+len(find)], find) {
|
||||
return i
|
||||
}
|
||||
i++
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
//not found
|
||||
func runeSliceBytes(in []rune) []byte {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
return unsafe.Slice((*byte)(unsafe.Pointer(&in[0])), len(in)*4)
|
||||
}
|
||||
|
||||
func indexOfRuneBytes(haystack []byte, find rune) int {
|
||||
needleRune := [1]rune{find}
|
||||
needle := runeSliceBytes(needleRune[:])
|
||||
start := 0
|
||||
for {
|
||||
idx := bytes.Index(haystack[start:], needle)
|
||||
if idx < 0 {
|
||||
return -1
|
||||
}
|
||||
idx += start
|
||||
if idx%4 == 0 {
|
||||
return idx / 4
|
||||
}
|
||||
start = idx + 1
|
||||
}
|
||||
}
|
||||
|
||||
func lastIndexOfRuneBytes(haystack []byte, find rune) int {
|
||||
needleRune := [1]rune{find}
|
||||
needle := runeSliceBytes(needleRune[:])
|
||||
end := len(haystack)
|
||||
for end >= 4 {
|
||||
idx := bytes.LastIndex(haystack[:end], needle)
|
||||
if idx < 0 {
|
||||
return -1
|
||||
}
|
||||
if idx%4 == 0 {
|
||||
return idx / 4
|
||||
}
|
||||
end = idx + 3
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
|
||||
+47
-18
@@ -64,14 +64,30 @@ func (s AsciiSearchValues) IndexOfAnyExcept(chars []rune) int {
|
||||
|
||||
// return the last index of our original vals values within the slice given
|
||||
func (s AsciiSearchValues) LastIndexOfAny(chars []rune) int {
|
||||
panic("not implemented")
|
||||
//TODO: this
|
||||
for i := len(chars) - 1; i >= 0; i-- {
|
||||
c := chars[i]
|
||||
if c > unicode.MaxASCII {
|
||||
continue
|
||||
}
|
||||
if s.set[c/64]&(1<<(c%64)) != 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// return the last index of our original vals values within the slice given
|
||||
func (s AsciiSearchValues) LastIndexOfAnyExcept(chars []rune) int {
|
||||
panic("not implemented")
|
||||
//TODO: this
|
||||
for i := len(chars) - 1; i >= 0; i-- {
|
||||
c := chars[i]
|
||||
if c > unicode.MaxASCII {
|
||||
return i
|
||||
}
|
||||
if s.set[c/64]&(1<<(c%64)) == 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
type RuneSearchValues struct {
|
||||
@@ -79,37 +95,40 @@ type RuneSearchValues struct {
|
||||
}
|
||||
|
||||
func newRuneSearchValues(vals []rune) RuneSearchValues {
|
||||
//TODO: pre-calc the stuff we need to make each IndexOf go faster
|
||||
return RuneSearchValues{vals: vals}
|
||||
|
||||
}
|
||||
|
||||
func NewRuneSearchValues(vals string) RuneSearchValues {
|
||||
return newRuneSearchValues([]rune(vals))
|
||||
}
|
||||
|
||||
// return the first index of our original vals values within the slice given
|
||||
func (s RuneSearchValues) IndexOfAny(chars []rune) int {
|
||||
//naive implementation
|
||||
//TODO: this
|
||||
return IndexOfAny(chars, s.vals)
|
||||
}
|
||||
|
||||
// return the first index of our original vals values within the slice given
|
||||
func (s RuneSearchValues) IndexOfAnyExcept(chars []rune) int {
|
||||
//TODO: this
|
||||
return IndexOfAnyExcept(chars, s.vals)
|
||||
}
|
||||
|
||||
// return the last index of our original vals values within the slice given
|
||||
func (s RuneSearchValues) LastIndexOfAny(chars []rune) int {
|
||||
panic("not implemented")
|
||||
if len(s.vals) == 1 {
|
||||
return LastIndexOfAny1(chars, s.vals[0])
|
||||
}
|
||||
for i := len(chars) - 1; i >= 0; i-- {
|
||||
if slices.Contains(s.vals, chars[i]) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// return the last index of our original vals values within the slice given
|
||||
func (s RuneSearchValues) LastIndexOfAnyExcept(chars []rune) int {
|
||||
panic("not implemented")
|
||||
//TODO: this
|
||||
for i := len(chars) - 1; i >= 0; i-- {
|
||||
if !slices.Contains(s.vals, chars[i]) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
type StringSearchValues struct {
|
||||
@@ -145,11 +164,21 @@ func NewStringSearchValues(vals [][]rune, ignoreCase bool) StringSearchValues {
|
||||
}
|
||||
|
||||
func (s StringSearchValues) StartsWith(chars []rune) int {
|
||||
panic("not implemented")
|
||||
for _, val := range s.vals {
|
||||
if StartsWith(chars, val) {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (s StringSearchValues) StartsWithIgnoreCase(chars []rune) int {
|
||||
panic("not implemented")
|
||||
for _, val := range s.vals {
|
||||
if StartsWithIgnoreCase(chars, val) {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (s StringSearchValues) IndexOfAny(in []rune) int {
|
||||
|
||||
+62
-17
@@ -44,8 +44,9 @@ type Group struct {
|
||||
type Capture struct {
|
||||
// the original string
|
||||
text *matchText
|
||||
// RuneIndex is the position in the underlying rune slice where the first character of
|
||||
// captured substring was found. Even if you pass in a string this will be in Runes.
|
||||
// RuneIndex is the rune index in the original input where the capture starts.
|
||||
// For string input this counts runes from the start of that string, not of
|
||||
// any internally sliced decode buffer.
|
||||
RuneIndex int
|
||||
// RuneLength is the number of runes in the captured substring.
|
||||
RuneLength int
|
||||
@@ -55,24 +56,46 @@ type matchText struct {
|
||||
runes []rune
|
||||
input string
|
||||
hasStringInput bool
|
||||
runeOffset int // original-string rune index of runes[0]
|
||||
byteOffset int // original-string byte index of runes[0]
|
||||
byteOffsets []int
|
||||
byteOffsetsReady bool
|
||||
}
|
||||
|
||||
// String returns the captured text as a String
|
||||
// String returns the captured text. For string input it is a slice of the
|
||||
// original haystack: it does not allocate and keeps the original string alive.
|
||||
func (c *Capture) String() string {
|
||||
return string(c.text.runes[c.RuneIndex : c.RuneIndex+c.RuneLength])
|
||||
if c.text == nil {
|
||||
return ""
|
||||
}
|
||||
if c.text.hasStringInput {
|
||||
start, length := c.ByteRange()
|
||||
return c.text.input[start : start+length]
|
||||
}
|
||||
start := c.runeSliceIndex()
|
||||
return string(c.text.runes[start : start+c.RuneLength])
|
||||
}
|
||||
|
||||
// Runes returns the captured text as a rune slice
|
||||
func (c *Capture) Runes() []rune {
|
||||
return c.text.runes[c.RuneIndex : c.RuneIndex+c.RuneLength]
|
||||
if c.text == nil {
|
||||
return nil
|
||||
}
|
||||
start := c.runeSliceIndex()
|
||||
return c.text.runes[start : start+c.RuneLength]
|
||||
}
|
||||
|
||||
func (c *Capture) runeSliceIndex() int {
|
||||
if c.text == nil {
|
||||
return c.RuneIndex
|
||||
}
|
||||
return c.RuneIndex - c.text.runeOffset
|
||||
}
|
||||
|
||||
// ByteRange returns the UTF-8 byte index and byte length of the captured
|
||||
// substring. The first call lazily caches byte offsets on shared match text,
|
||||
// so it is not safe to call concurrently with ByteRange on another capture
|
||||
// from the same match until the cache has been initialized.
|
||||
// substring. Matches returned to callers have offsets computed when the match
|
||||
// is tidied, so concurrent ByteRange/String on captures of the same match is
|
||||
// then safe. Internal match objects may still initialize the cache on first use.
|
||||
func (c *Capture) ByteRange() (index, length int) {
|
||||
if c.text == nil {
|
||||
return c.RuneIndex, c.RuneLength
|
||||
@@ -85,24 +108,40 @@ func newMatchText(r []rune) *matchText {
|
||||
}
|
||||
|
||||
func newStringMatchText(input string, r []rune) *matchText {
|
||||
return &matchText{runes: r, input: input, hasStringInput: true}
|
||||
return newStringMatchTextAt(input, r, 0, 0)
|
||||
}
|
||||
|
||||
func newStringMatchTextAt(input string, r []rune, runeOffset, byteOffset int) *matchText {
|
||||
return &matchText{
|
||||
runes: r,
|
||||
input: input,
|
||||
hasStringInput: true,
|
||||
runeOffset: runeOffset,
|
||||
byteOffset: byteOffset,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *matchText) ensureByteOffsets() {
|
||||
if t == nil || t.byteOffsetsReady {
|
||||
return
|
||||
}
|
||||
t.byteOffsets = t.buildByteOffsets()
|
||||
t.byteOffsetsReady = true
|
||||
}
|
||||
|
||||
func (t *matchText) byteRange(runeIndex, runeLength int) (int, int) {
|
||||
if !t.byteOffsetsReady {
|
||||
t.byteOffsets = t.buildByteOffsets()
|
||||
t.byteOffsetsReady = true
|
||||
}
|
||||
localRuneIndex := runeIndex - t.runeOffset
|
||||
t.ensureByteOffsets()
|
||||
if t.byteOffsets == nil {
|
||||
return runeIndex, runeLength
|
||||
return t.byteOffset + localRuneIndex, runeLength
|
||||
}
|
||||
byteIndex := t.byteOffsets[runeIndex]
|
||||
return byteIndex, t.byteOffsets[runeIndex+runeLength] - byteIndex
|
||||
byteIndex := t.byteOffsets[localRuneIndex]
|
||||
return t.byteOffset + byteIndex, t.byteOffsets[localRuneIndex+runeLength] - byteIndex
|
||||
}
|
||||
|
||||
func (t *matchText) buildByteOffsets() []int {
|
||||
if t.hasStringInput {
|
||||
return stringByteOffsets(t.input)
|
||||
return stringByteOffsets(t.input[t.byteOffset:])
|
||||
}
|
||||
return runeByteOffsets(t.runes)
|
||||
}
|
||||
@@ -199,6 +238,9 @@ func (m *Match) tidy(textpos int) {
|
||||
m.capcount = m.matchcount[0]
|
||||
//copy our root capture to the list
|
||||
m.Captures = []Capture{m.Capture}
|
||||
if m.text != nil && m.text.hasStringInput {
|
||||
m.text.ensureByteOffsets()
|
||||
}
|
||||
|
||||
if m.balancing {
|
||||
// The idea here is that we want to compact all of our unbalanced captures. To do that we
|
||||
@@ -415,6 +457,9 @@ func newCapture(text *matchText, runeIndex, runeLength int) Capture {
|
||||
}
|
||||
|
||||
func setCaptureFields(c *Capture, runeIndex, runeLength int) {
|
||||
if c.text != nil {
|
||||
runeIndex += c.text.runeOffset
|
||||
}
|
||||
c.RuneIndex = runeIndex
|
||||
c.RuneLength = runeLength
|
||||
}
|
||||
|
||||
+14
-1
@@ -5,6 +5,7 @@ var (
|
||||
DefaultUnmarshalOptions = None
|
||||
// DefaultOptimizationOptions controls the default memory/performance trade-offs used by Compile.
|
||||
DefaultOptimizationOptions = OptimizationOptions{
|
||||
MaxBacktrackingStackSize: 100000,
|
||||
MaxCachedRuneBufferLength: 256 << 10,
|
||||
MaxCachedReplaceBufferLength: 256 << 10,
|
||||
MaxCachedReplacerDataEntries: 16,
|
||||
@@ -36,13 +37,17 @@ const (
|
||||
Unicode RegexOptions = 0x0400 // "u"
|
||||
)
|
||||
|
||||
// OptimizationOptions controls optional runtime caches and compile-time fast paths.
|
||||
// OptimizationOptions controls runtime limits, optional caches, and compile-time fast paths.
|
||||
//
|
||||
// For MaxBacktrackingStackSize, negative values allow unbounded growth.
|
||||
// For replacement data cache size fields, 0 disables persistent retention and
|
||||
// -1 means unbounded. For pooled buffer cache size fields, 0 disables pooling
|
||||
// and -1 allows all built-in size classes.
|
||||
// Defaults are intentionally bounded so Compile is safe for mixed-cardinality inputs.
|
||||
type OptimizationOptions struct {
|
||||
// MaxBacktrackingStackSize limits the number of integer slots used by a match's backtracking stack.
|
||||
// Negative values disable the limit.
|
||||
MaxBacktrackingStackSize int
|
||||
// MaxCachedRuneBufferLength limits retained string-to-rune buffers in the shared size-classed pool.
|
||||
MaxCachedRuneBufferLength int
|
||||
// MaxCachedReplaceBufferLength limits retained replacement output buffers in the shared size-classed pool.
|
||||
@@ -100,6 +105,14 @@ func newCompileConfig(options []CompileOption) compileConfig {
|
||||
return c
|
||||
}
|
||||
|
||||
// OptionMaxBacktrackingStackSize limits the number of integer slots used by a match's backtracking stack.
|
||||
// Negative values disable the limit. A match that exceeds the limit returns ErrBacktrackingStackLimit.
|
||||
func OptionMaxBacktrackingStackSize(n int) CompileOption {
|
||||
return compileOptionFunc(func(c *compileConfig) {
|
||||
c.optimizations.MaxBacktrackingStackSize = n
|
||||
})
|
||||
}
|
||||
|
||||
// OptionMaxCachedRuneBufferLength limits retained string-to-rune buffers in the shared size-classed pool.
|
||||
func OptionMaxCachedRuneBufferLength(n int) CompileOption {
|
||||
return compileOptionFunc(func(c *compileConfig) {
|
||||
|
||||
+64
-111
@@ -25,6 +25,8 @@ import (
|
||||
var (
|
||||
// DefaultMatchTimeout used when running regexp matches -- "forever"
|
||||
DefaultMatchTimeout = time.Duration(math.MaxInt64)
|
||||
// ErrBacktrackingStackLimit is returned when a match exceeds its configured backtracking stack size.
|
||||
ErrBacktrackingStackLimit = errors.New("regexp2: maximum backtracking stack size exceeded")
|
||||
)
|
||||
|
||||
// Regexp is the representation of a compiled regular expression.
|
||||
@@ -59,7 +61,12 @@ type Regexp struct {
|
||||
// hook points to override runner functions
|
||||
findFirstChar func(r *Runner) bool
|
||||
execute func(r *Runner) error
|
||||
executeQuick func(r *Runner) error
|
||||
stringPrefixFilter StringPrefixFilter
|
||||
quickCode *syntax.Code // bool-only program with unobservable captures removed
|
||||
// leftContextRunes is used when code is nil (registered engines).
|
||||
// The interpreter reads the same value from code.LeftContextRunes.
|
||||
leftContextRunes int
|
||||
}
|
||||
|
||||
// Compile parses a regular expression and returns, if successful,
|
||||
@@ -107,6 +114,7 @@ func compile(expr string, c compileConfig) (*Regexp, error) {
|
||||
capslist: tree.Caplist,
|
||||
capsize: code.Capsize,
|
||||
code: code,
|
||||
quickCode: makeQuickCode(code),
|
||||
MatchTimeout: DefaultMatchTimeout,
|
||||
optimizations: c.optimizations,
|
||||
}
|
||||
@@ -115,6 +123,18 @@ func compile(expr string, c compileConfig) (*Regexp, error) {
|
||||
return re, nil
|
||||
}
|
||||
|
||||
func makeQuickCode(code *syntax.Code) *syntax.Code {
|
||||
if code == nil || len(code.QuickCodes) == 0 {
|
||||
return nil
|
||||
}
|
||||
quick := *code
|
||||
quick.Codes = code.QuickCodes
|
||||
quick.Dispatches = code.QuickDispatches
|
||||
quick.QuickCodes = nil
|
||||
quick.QuickDispatches = nil
|
||||
return &quick
|
||||
}
|
||||
|
||||
// MustCompile is like Compile but panics if the expression cannot be parsed.
|
||||
// It simplifies safe initialization of global variables holding compiled regular
|
||||
// expressions.
|
||||
@@ -225,17 +245,12 @@ func (re *Regexp) FindStringMatch(s string) (*Match, error) {
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
r, runeStart := re.getRunesAndStart(s, startAt)
|
||||
if runeStart < 0 {
|
||||
runeStart = 0
|
||||
}
|
||||
return re.run(false, runeStart, r, newStringMatchText(s, r))
|
||||
return re.findDecodedStringMatch(s, startAt)
|
||||
}
|
||||
|
||||
// FindRunesMatch searches the input rune slice for a Regexp match
|
||||
func (re *Regexp) FindRunesMatch(r []rune) (*Match, error) {
|
||||
return re.run(false, -1, r, newMatchText(r))
|
||||
return re.run(false, -1, -1, r, newMatchText(r))
|
||||
}
|
||||
|
||||
// FindStringMatchStartingAt searches the input string for a Regexp match starting at the startAt index
|
||||
@@ -247,19 +262,22 @@ func (re *Regexp) FindStringMatchStartingAt(s string, startAt int) (*Match, erro
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return re.findDecodedStringMatch(s, startAt)
|
||||
}
|
||||
|
||||
r, startAt := re.getRunesAndStart(s, startAt)
|
||||
if startAt == -1 {
|
||||
// we didn't find our start index in the string -- that's a problem
|
||||
return nil, errors.New("startAt must align to the start of a valid rune in the input string")
|
||||
}
|
||||
|
||||
return re.run(false, startAt, r, newStringMatchText(s, r))
|
||||
func (re *Regexp) findDecodedStringMatch(s string, startAt int) (*Match, error) {
|
||||
// Returned matches retain their rune data, so this path must not consume a
|
||||
// pooled buffer that can never be returned.
|
||||
d := re.decodeStringInput(s, startAt, false)
|
||||
runner := re.getRunner()
|
||||
defer re.putRunner(runner)
|
||||
text := newStringMatchTextAt(s, d.runes, d.runeOffset, d.byteOffset)
|
||||
return runner.scan(d.runes, text, d.runeStart, -1, false, re.MatchTimeout)
|
||||
}
|
||||
|
||||
// FindRunesMatchStartingAt searches the input rune slice for a Regexp match starting at the startAt index
|
||||
func (re *Regexp) FindRunesMatchStartingAt(r []rune, startAt int) (*Match, error) {
|
||||
return re.run(false, startAt, r, newMatchText(r))
|
||||
return re.run(false, startAt, -1, r, newMatchText(r))
|
||||
}
|
||||
|
||||
// FindAllStringIndex returns a slice of byte index pairs identifying all
|
||||
@@ -277,33 +295,22 @@ func (re *Regexp) FindAllStringIndex(s string, n int) ([][]int, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
d := re.decodeStringInput(s, startAt, true)
|
||||
runner := re.getRunner()
|
||||
var input []rune
|
||||
var pooledInput *[]rune
|
||||
runeStart := 0
|
||||
if startAt == 0 {
|
||||
input, pooledInput = runner.decodeString(s)
|
||||
} else {
|
||||
input, runeStart, pooledInput = runner.decodeStringWithStart(s, startAt)
|
||||
}
|
||||
defer func() {
|
||||
re.putRunner(runner)
|
||||
if pooledInput != nil {
|
||||
*pooledInput = input
|
||||
pooledRuneBuffers.put(pooledInput)
|
||||
}
|
||||
d.release()
|
||||
}()
|
||||
|
||||
if runeStart < 0 {
|
||||
runeStart = 0
|
||||
}
|
||||
|
||||
byteOffsets := newStringByteMapper(s)
|
||||
return re.findAllRunesIndex(runner, input, runeStart, n, func(runeIndex, runeLength int) (int, int) {
|
||||
if re.quickCode != nil {
|
||||
runner.code = re.quickCode
|
||||
}
|
||||
return re.findAllRunesIndex(runner, d.runes, d.runeStart, n, func(runeIndex, runeLength int) (int, int) {
|
||||
if byteOffsets == nil {
|
||||
return runeIndex, runeIndex + runeLength
|
||||
return d.byteOffset + runeIndex, d.byteOffset + runeIndex + runeLength
|
||||
}
|
||||
return byteOffsets.byteIndex(runeIndex), byteOffsets.byteIndex(runeIndex + runeLength)
|
||||
start := runeIndex + d.runeOffset
|
||||
return byteOffsets.byteIndex(start), byteOffsets.byteIndex(start + runeLength)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -321,6 +328,9 @@ func (re *Regexp) FindAllRunesIndex(r []rune, n int) ([][]int, error) {
|
||||
if re.RightToLeft() {
|
||||
startAt = len(r)
|
||||
}
|
||||
if re.quickCode != nil {
|
||||
runner.code = re.quickCode
|
||||
}
|
||||
return re.findAllRunesIndex(runner, r, startAt, n, func(runeIndex, runeLength int) (int, int) {
|
||||
return runeIndex, runeIndex + runeLength
|
||||
})
|
||||
@@ -335,8 +345,9 @@ func (re *Regexp) findAllRunesIndex(runner *Runner, input []rune, startAt, n int
|
||||
}
|
||||
|
||||
prevEnd := -1
|
||||
previousMatchLength := -1
|
||||
for n != 0 {
|
||||
m, err := runner.scan(input, nil, startAt, true, re.MatchTimeout)
|
||||
m, err := runner.scan(input, nil, startAt, previousMatchLength, true, re.MatchTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -344,34 +355,19 @@ func (re *Regexp) findAllRunesIndex(runner *Runner, input []rune, startAt, n int
|
||||
break
|
||||
}
|
||||
|
||||
if m.RuneLength != 0 || m.RuneIndex != prevEnd {
|
||||
start, end := makeIndex(m.RuneIndex, m.RuneLength)
|
||||
localIndex := m.runeSliceIndex()
|
||||
if m.RuneLength != 0 || localIndex != prevEnd {
|
||||
start, end := makeIndex(localIndex, m.RuneLength)
|
||||
flat = append(flat, start, end)
|
||||
out = append(out, flat[len(flat)-2:len(flat):len(flat)])
|
||||
prevEnd = m.RuneIndex + m.RuneLength
|
||||
prevEnd = localIndex + m.RuneLength
|
||||
if n > 0 {
|
||||
n--
|
||||
}
|
||||
}
|
||||
|
||||
startAt = m.textpos
|
||||
if m.RuneLength == 0 {
|
||||
if re.RightToLeft() {
|
||||
if m.textpos == 0 {
|
||||
break
|
||||
}
|
||||
if startAt == m.textstart {
|
||||
startAt--
|
||||
}
|
||||
} else {
|
||||
if m.textpos == len(input) {
|
||||
break
|
||||
}
|
||||
if startAt == m.textstart {
|
||||
startAt++
|
||||
}
|
||||
}
|
||||
}
|
||||
previousMatchLength = m.RuneLength
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -420,29 +416,7 @@ func (re *Regexp) FindNextMatch(m *Match) (*Match, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// If previous match was empty, advance by one before matching to prevent
|
||||
// infinite loop
|
||||
startAt := m.textpos
|
||||
if m.RuneLength == 0 {
|
||||
if re.RightToLeft() {
|
||||
if m.textpos == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if startAt == m.textstart {
|
||||
startAt--
|
||||
}
|
||||
} else {
|
||||
if m.textpos == len(m.text.runes) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if startAt == m.textstart {
|
||||
startAt++
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return re.run(false, startAt, m.text.runes, m.text)
|
||||
return re.run(false, m.textpos, m.RuneLength, m.text.runes, m.text)
|
||||
}
|
||||
|
||||
// MatchString return true if the string matches the regex
|
||||
@@ -469,12 +443,16 @@ func (re *Regexp) matchStringAt(s string, startAt int) (bool, error) {
|
||||
var pooledInput *[]rune
|
||||
runeStart := 0
|
||||
if startAt <= 0 {
|
||||
input, pooledInput = runner.decodeString(s)
|
||||
// Common path: decode the whole string without start/offset work.
|
||||
input, pooledInput = decodeString(s, re.optimizations.MaxCachedRuneBufferLength)
|
||||
if re.RightToLeft() {
|
||||
runeStart = len(input)
|
||||
}
|
||||
} else {
|
||||
input, runeStart, pooledInput = runner.decodeStringWithStart(s, startAt)
|
||||
d := decodeInput(s, startAt, re.decodeFrom(s, startAt), re.optimizations.MaxCachedRuneBufferLength, false)
|
||||
input = d.runes
|
||||
pooledInput = d.pooled
|
||||
runeStart = d.runeStart
|
||||
if runeStart < 0 {
|
||||
runeStart = 0
|
||||
}
|
||||
@@ -486,46 +464,21 @@ func (re *Regexp) matchStringAt(s string, startAt int) (bool, error) {
|
||||
pooledRuneBuffers.put(pooledInput)
|
||||
}
|
||||
}()
|
||||
if re.quickCode != nil {
|
||||
runner.code = re.quickCode
|
||||
}
|
||||
|
||||
m, err := runner.scan(input, nil, runeStart, true, re.MatchTimeout)
|
||||
m, err := runner.scan(input, nil, runeStart, -1, true, re.MatchTimeout)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return m != nil, nil
|
||||
}
|
||||
|
||||
func (re *Regexp) getRunesAndStart(s string, startAt int) ([]rune, int) {
|
||||
if startAt < 0 {
|
||||
if re.RightToLeft() {
|
||||
r := getRunes(s)
|
||||
return r, len(r)
|
||||
}
|
||||
return getRunes(s), 0
|
||||
}
|
||||
ret := make([]rune, len(s))
|
||||
i := 0
|
||||
runeIdx := -1
|
||||
for strIdx, r := range s {
|
||||
if strIdx == startAt {
|
||||
runeIdx = i
|
||||
}
|
||||
ret[i] = r
|
||||
i++
|
||||
}
|
||||
if startAt == len(s) {
|
||||
runeIdx = i
|
||||
}
|
||||
return ret[:i], runeIdx
|
||||
}
|
||||
|
||||
func getRunes(s string) []rune {
|
||||
return []rune(s)
|
||||
}
|
||||
|
||||
// MatchRunes return true if the runes matches the regex
|
||||
// error will be set if a timeout occurs
|
||||
func (re *Regexp) MatchRunes(r []rune) (bool, error) {
|
||||
m, err := re.run(true, -1, r, nil)
|
||||
m, err := re.run(true, -1, -1, r, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
+24
-7
@@ -11,13 +11,23 @@ type RuntimeEngineData struct {
|
||||
CapSize int // size of the capture array
|
||||
FindFirstChar func(*Runner) bool // generated candidate search
|
||||
Execute func(*Runner) error
|
||||
StringPrefixFilter StringPrefixFilter // optional pre-decode candidate search for string input
|
||||
ExecuteQuick func(*Runner) error // optional bool-only execution with unobservable captures removed
|
||||
StringPrefixFilter StringPrefixFilter // optional pre-decode candidate search for string input
|
||||
// LeftContextKnown reports that LeftContextRunes was computed by the
|
||||
// code generator. If it is false, decoded string input is never sliced;
|
||||
// older generated engines omit the field and must keep the full string.
|
||||
LeftContextKnown bool
|
||||
// LeftContextRunes is how many runes before a candidate start matching
|
||||
// may inspect. 0 means none, 1 means a single previous rune, and -1
|
||||
// means do not slice (lookbehind or \G). Ignored unless LeftContextKnown.
|
||||
LeftContextRunes int
|
||||
}
|
||||
|
||||
type cacheKey struct {
|
||||
pattern string
|
||||
opt RegexOptions
|
||||
maintainCaptureOrder bool
|
||||
pattern string
|
||||
opt RegexOptions
|
||||
maintainCaptureOrder bool
|
||||
maxBacktrackingStackSize int
|
||||
}
|
||||
|
||||
func RegisterEngine(pattern string, engine RuntimeEngineData, options ...CompileOption) {
|
||||
@@ -28,6 +38,10 @@ func RegisterEngine(pattern string, engine RuntimeEngineData, options ...Compile
|
||||
}
|
||||
|
||||
func newEngineRegexp(pattern string, c compileConfig, engine RuntimeEngineData) *Regexp {
|
||||
leftContext := -1
|
||||
if engine.LeftContextKnown {
|
||||
leftContext = engine.LeftContextRunes
|
||||
}
|
||||
re := &Regexp{
|
||||
pattern: pattern,
|
||||
options: c.regexOptions,
|
||||
@@ -40,7 +54,9 @@ func newEngineRegexp(pattern string, c compileConfig, engine RuntimeEngineData)
|
||||
optimizations: c.optimizations,
|
||||
findFirstChar: engine.FindFirstChar,
|
||||
execute: engine.Execute,
|
||||
executeQuick: engine.ExecuteQuick,
|
||||
stringPrefixFilter: engine.StringPrefixFilter,
|
||||
leftContextRunes: leftContext,
|
||||
}
|
||||
re.initCaches()
|
||||
return re
|
||||
@@ -58,9 +74,10 @@ func getEngineRegexp(pattern string, c compileConfig) *Regexp {
|
||||
|
||||
func cacheKeyFromConfig(pattern string, c compileConfig) cacheKey {
|
||||
return cacheKey{
|
||||
pattern: pattern,
|
||||
opt: c.regexOptions,
|
||||
maintainCaptureOrder: c.maintainCaptureOrder,
|
||||
pattern: pattern,
|
||||
opt: c.regexOptions,
|
||||
maintainCaptureOrder: c.maintainCaptureOrder,
|
||||
maxBacktrackingStackSize: c.optimizations.MaxBacktrackingStackSize,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+47
-49
@@ -87,15 +87,15 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator,
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
text := m.text.runes
|
||||
|
||||
if !regex.RightToLeft() {
|
||||
prevat := 0
|
||||
for m != nil {
|
||||
if m.RuneIndex != prevat {
|
||||
buf.WriteString(string(text[prevat:m.RuneIndex]))
|
||||
start, end := matchInputSpan(m)
|
||||
if start > prevat {
|
||||
buf.WriteString(input[prevat:start])
|
||||
}
|
||||
prevat = m.RuneIndex + m.RuneLength
|
||||
prevat = end
|
||||
buf.WriteString(evaluator(*m))
|
||||
|
||||
count--
|
||||
@@ -108,18 +108,19 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator,
|
||||
}
|
||||
}
|
||||
|
||||
if prevat < len(text) {
|
||||
buf.WriteString(string(text[prevat:]))
|
||||
if prevat < len(input) {
|
||||
buf.WriteString(input[prevat:])
|
||||
}
|
||||
} else {
|
||||
prevat := len(text)
|
||||
prevat := len(input)
|
||||
var al []string
|
||||
|
||||
for m != nil {
|
||||
if m.RuneIndex+m.RuneLength != prevat {
|
||||
al = append(al, string(text[m.RuneIndex+m.RuneLength:prevat]))
|
||||
start, end := matchInputSpan(m)
|
||||
if end < prevat {
|
||||
al = append(al, input[end:prevat])
|
||||
}
|
||||
prevat = m.RuneIndex
|
||||
prevat = start
|
||||
al = append(al, evaluator(*m))
|
||||
|
||||
count--
|
||||
@@ -133,7 +134,7 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator,
|
||||
}
|
||||
|
||||
if prevat > 0 {
|
||||
buf.WriteString(string(text[:prevat]))
|
||||
buf.WriteString(input[:prevat])
|
||||
}
|
||||
|
||||
for i := len(al) - 1; i >= 0; i-- {
|
||||
@@ -144,20 +145,29 @@ func replace(regex *Regexp, data *syntax.ReplacerData, evaluator MatchEvaluator,
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
// matchInputSpan returns the UTF-8 byte range of m in its original string input.
|
||||
func matchInputSpan(m *Match) (start, end int) {
|
||||
if m.text != nil && m.text.hasStringInput {
|
||||
start, length := m.ByteRange()
|
||||
return start, start + length
|
||||
}
|
||||
return m.RuneIndex, m.RuneIndex + m.RuneLength
|
||||
}
|
||||
|
||||
func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, startAt, count int) (string, error) {
|
||||
if startAt > len(input) {
|
||||
return "", errors.New("startAt must be less than the length of the input string")
|
||||
}
|
||||
|
||||
runner := regex.getRunner()
|
||||
text, runeStart, pooledText := runner.decodeStringWithStart(input, startAt)
|
||||
d := decodeInput(input, startAt, 0, regex.optimizations.MaxCachedRuneBufferLength, false)
|
||||
text := d.runes
|
||||
textInfo := newStringMatchText(input, text)
|
||||
defer func() {
|
||||
regex.putRunner(runner)
|
||||
if pooledText != nil {
|
||||
pooledRuneBuffers.put(pooledText)
|
||||
}
|
||||
d.release()
|
||||
}()
|
||||
runeStart := d.runeStart
|
||||
if startAt >= 0 && runeStart < 0 {
|
||||
return "", errors.New("startAt must align to the start of a valid rune in the input string")
|
||||
}
|
||||
@@ -165,7 +175,7 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
runeStart = 0
|
||||
}
|
||||
|
||||
m, err := runner.scan(text, textInfo, runeStart, true, regex.MatchTimeout)
|
||||
m, err := runner.scan(text, textInfo, runeStart, -1, true, regex.MatchTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -184,10 +194,11 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
compactBalancedMatches(m)
|
||||
}
|
||||
|
||||
if m.RuneIndex != prevat {
|
||||
writeRunes(buf, text, prevat, m.RuneIndex)
|
||||
local := m.runeSliceIndex()
|
||||
if local != prevat {
|
||||
writeRunes(buf, text, prevat, local)
|
||||
}
|
||||
prevat = m.RuneIndex + m.RuneLength
|
||||
prevat = local + m.RuneLength
|
||||
replacementImpl(data, buf, m)
|
||||
|
||||
count--
|
||||
@@ -195,15 +206,7 @@ func replaceRunnerLTR(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
break
|
||||
}
|
||||
|
||||
scanStart := m.textpos
|
||||
if m.RuneLength == 0 {
|
||||
if scanStart >= len(text) {
|
||||
break
|
||||
}
|
||||
scanStart++
|
||||
}
|
||||
|
||||
m, err = runner.scan(text, textInfo, scanStart, true, regex.MatchTimeout)
|
||||
m, err = runner.scan(text, textInfo, m.textpos, m.RuneLength, true, regex.MatchTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -221,14 +224,14 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
}
|
||||
|
||||
runner := regex.getRunner()
|
||||
text, runeStart, pooledText := runner.decodeStringWithStart(input, startAt)
|
||||
d := decodeInput(input, startAt, 0, regex.optimizations.MaxCachedRuneBufferLength, false)
|
||||
text := d.runes
|
||||
textInfo := newStringMatchText(input, text)
|
||||
defer func() {
|
||||
regex.putRunner(runner)
|
||||
if pooledText != nil {
|
||||
pooledRuneBuffers.put(pooledText)
|
||||
}
|
||||
d.release()
|
||||
}()
|
||||
runeStart := d.runeStart
|
||||
if startAt >= 0 && runeStart < 0 {
|
||||
return "", errors.New("startAt must align to the start of a valid rune in the input string")
|
||||
}
|
||||
@@ -236,7 +239,7 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
runeStart = len(text)
|
||||
}
|
||||
|
||||
m, err := runner.scan(text, textInfo, runeStart, true, regex.MatchTimeout)
|
||||
m, err := runner.scan(text, textInfo, runeStart, -1, true, regex.MatchTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -257,10 +260,11 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
compactBalancedMatches(m)
|
||||
}
|
||||
|
||||
if m.RuneIndex+m.RuneLength != prevat {
|
||||
al = append(al, string(text[m.RuneIndex+m.RuneLength:prevat]))
|
||||
local := m.runeSliceIndex()
|
||||
if local+m.RuneLength != prevat {
|
||||
al = append(al, string(text[local+m.RuneLength:prevat]))
|
||||
}
|
||||
prevat = m.RuneIndex
|
||||
prevat = local
|
||||
replacementImplRTL(data, &al, m)
|
||||
|
||||
count--
|
||||
@@ -268,15 +272,7 @@ func replaceRunnerRTL(regex *Regexp, data *syntax.ReplacerData, input string, st
|
||||
break
|
||||
}
|
||||
|
||||
scanStart := m.textpos
|
||||
if m.RuneLength == 0 {
|
||||
if scanStart <= 0 {
|
||||
break
|
||||
}
|
||||
scanStart--
|
||||
}
|
||||
|
||||
m, err = runner.scan(text, textInfo, scanStart, true, regex.MatchTimeout)
|
||||
m, err = runner.scan(text, textInfo, m.textpos, m.RuneLength, true, regex.MatchTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -303,11 +299,12 @@ func replacementImpl(data *syntax.ReplacerData, buf *bytes.Buffer, m *Match) {
|
||||
} else {
|
||||
switch -replaceSpecials - 1 - r { // special insertion patterns
|
||||
case replaceLeftPortion:
|
||||
for i := 0; i < m.RuneIndex; i++ {
|
||||
end := m.runeSliceIndex()
|
||||
for i := 0; i < end; i++ {
|
||||
buf.WriteRune(m.text.runes[i])
|
||||
}
|
||||
case replaceRightPortion:
|
||||
for i := m.RuneIndex + m.RuneLength; i < len(m.text.runes); i++ {
|
||||
for i := m.runeSliceIndex() + m.RuneLength; i < len(m.text.runes); i++ {
|
||||
buf.WriteRune(m.text.runes[i])
|
||||
}
|
||||
case replaceLastGroup:
|
||||
@@ -335,11 +332,12 @@ func replacementImplRTL(data *syntax.ReplacerData, al *[]string, m *Match) {
|
||||
} else {
|
||||
switch -replaceSpecials - 1 - r { // special insertion patterns
|
||||
case replaceLeftPortion:
|
||||
for i := 0; i < m.RuneIndex; i++ {
|
||||
end := m.runeSliceIndex()
|
||||
for i := 0; i < end; i++ {
|
||||
buf.WriteRune(m.text.runes[i])
|
||||
}
|
||||
case replaceRightPortion:
|
||||
for i := m.RuneIndex + m.RuneLength; i < len(m.text.runes); i++ {
|
||||
for i := m.runeSliceIndex() + m.RuneLength; i < len(m.text.runes); i++ {
|
||||
buf.WriteRune(m.text.runes[i])
|
||||
}
|
||||
case replaceLastGroup:
|
||||
|
||||
+239
-75
@@ -74,8 +74,10 @@ type Runner struct {
|
||||
//
|
||||
// quick is usually false, but can be true to not return matches, just put it in caches.
|
||||
// textstart is -1 to start at the "beginning" (depending on Right-To-Left), otherwise an index in input.
|
||||
// previousMatchLength is -1 for an initial scan. A zero value advances the current scan position while
|
||||
// preserving textstart for anchors such as \G.
|
||||
// textInfo is nil for quick scans that do not need returned capture text metadata.
|
||||
func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchText) (*Match, error) {
|
||||
func (re *Regexp) run(quick bool, textstart, previousMatchLength int, input []rune, textInfo *matchText) (*Match, error) {
|
||||
|
||||
// get a cached runner
|
||||
runner := re.getRunner()
|
||||
@@ -88,8 +90,11 @@ func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchTe
|
||||
textstart = 0
|
||||
}
|
||||
}
|
||||
if quick && textInfo == nil && re.quickCode != nil {
|
||||
runner.code = re.quickCode
|
||||
}
|
||||
|
||||
return runner.scan(input, textInfo, textstart, quick, re.MatchTimeout)
|
||||
return runner.scan(input, textInfo, textstart, previousMatchLength, quick, re.MatchTimeout)
|
||||
}
|
||||
|
||||
// Scans the string to find the first match. Uses the Match object
|
||||
@@ -108,13 +113,16 @@ func (re *Regexp) run(quick bool, textstart int, input []rune, textInfo *matchTe
|
||||
// used as a boolean result and capture text is intentionally unavailable. If
|
||||
// we collapsed down to just textInfo it would "escape" and hit the GC for fast
|
||||
// scans without captures.
|
||||
func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool, timeout time.Duration) (*Match, error) {
|
||||
func (r *Runner) scan(rt []rune, textInfo *matchText, textstart, previousMatchLength int, quick bool, timeout time.Duration) (*Match, error) {
|
||||
r.timeout = timeout
|
||||
r.ignoreTimeout = (time.Duration(math.MaxInt64) == timeout)
|
||||
r.debug = r.re.Debug()
|
||||
r.Runtextstart = textstart
|
||||
r.Runtext = rt
|
||||
r.Runtextend = len(rt)
|
||||
// Some internal callers use quick match tidying while still consuming
|
||||
// capture data (notably replacement). Capture elision is only safe when no
|
||||
// match text metadata was requested.
|
||||
|
||||
stoppos := r.Runtextend
|
||||
bump := 1
|
||||
@@ -130,6 +138,9 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool,
|
||||
// setup our scanner functions
|
||||
findFirstChar := r.re.findFirstChar
|
||||
execute := r.re.execute
|
||||
if quick && textInfo == nil && r.re.executeQuick != nil {
|
||||
execute = r.re.executeQuick
|
||||
}
|
||||
if findFirstChar == nil {
|
||||
findFirstChar = findFirstCharDefault
|
||||
}
|
||||
@@ -144,6 +155,16 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool,
|
||||
|
||||
r.initMatch(textInfo)
|
||||
|
||||
// An empty previous match must not be returned again. Keep Runtextstart at
|
||||
// the previous match position for \G, but move the candidate scan position.
|
||||
if previousMatchLength == 0 {
|
||||
if r.Runtextpos == stoppos {
|
||||
r.tidyMatch(true)
|
||||
return nil, nil
|
||||
}
|
||||
r.Runtextpos += bump
|
||||
}
|
||||
|
||||
r.startTimeoutWatch()
|
||||
for {
|
||||
if minRequiredLength > 0 {
|
||||
@@ -208,8 +229,9 @@ func (r *Runner) scan(rt []rune, textInfo *matchText, textstart int, quick bool,
|
||||
|
||||
func executeDefault(r *Runner) error {
|
||||
|
||||
r.goTo(0)
|
||||
|
||||
if err := r.goTo(0); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
|
||||
if r.debug {
|
||||
@@ -230,7 +252,59 @@ func executeDefault(r *Runner) error {
|
||||
//noop
|
||||
|
||||
case syntax.Goto:
|
||||
r.goTo(r.operand(0))
|
||||
if err := r.goTo(r.operand(0)); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
|
||||
case syntax.Dispatch:
|
||||
// Dispatch only peeks at the next rune. It consumes it after finding a
|
||||
// matching branch, so a failed dispatch leaves the input position alone.
|
||||
if r.forwardchars() < 1 {
|
||||
break
|
||||
}
|
||||
// Pick the next rune in the current execution direction.
|
||||
pos := r.Runtextpos
|
||||
if r.rightToLeft {
|
||||
pos--
|
||||
}
|
||||
ch := r.Runtext[pos]
|
||||
tableIndex := r.operand(0)
|
||||
table := &r.code.Dispatches[tableIndex]
|
||||
branch := -1
|
||||
if ch >= 0 && ch < 128 {
|
||||
if table.ASCII != nil {
|
||||
// Larger dispatches use a direct ASCII branch lookup.
|
||||
branch = int(table.ASCII[ch]) - 1
|
||||
} else {
|
||||
// Smaller dispatches use two compact ASCII bitmasks per set.
|
||||
word := int(ch >> 6)
|
||||
bit := uint64(1) << (ch & 63)
|
||||
for i := range table.Sets {
|
||||
if table.ASCIIMasks[i*2+word]&bit != 0 {
|
||||
branch = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-ASCII runes fall back to the complete character sets.
|
||||
for i, setIndex := range table.Sets {
|
||||
if r.code.Sets[setIndex].CharIn(ch) {
|
||||
branch = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if branch < 0 {
|
||||
break
|
||||
}
|
||||
// The selected branch starts with this rune, so consume it and jump
|
||||
// directly to the rest of that branch.
|
||||
r.Runtextpos += r.bump()
|
||||
if err := r.goTo(table.Branches[branch]); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
|
||||
case syntax.Testref:
|
||||
@@ -248,7 +322,9 @@ func executeDefault(r *Runner) error {
|
||||
case syntax.Lazybranch | syntax.Back:
|
||||
r.trackPop()
|
||||
r.textto(r.trackPeek())
|
||||
r.goTo(r.operand(0))
|
||||
if err := r.goTo(r.operand(0)); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
|
||||
case syntax.Setmark:
|
||||
@@ -307,9 +383,11 @@ func executeDefault(r *Runner) error {
|
||||
matched := r.textPos() - r.stackPeek()
|
||||
|
||||
if matched != 0 { // Nonempty match -> loop now
|
||||
r.trackPush2(r.stackPeek(), r.textPos()) // Save old mark, textpos
|
||||
r.stackPush(r.textPos()) // Make new mark
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.trackPush2(r.stackPeek(), r.textPos()) // Save old mark, textpos
|
||||
r.stackPush(r.textPos()) // Make new mark
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
} else { // Empty match -> straight now
|
||||
r.trackPushNeg1(r.stackPeek()) // Save old mark
|
||||
r.advance(1) // Straight
|
||||
@@ -364,10 +442,12 @@ func executeDefault(r *Runner) error {
|
||||
|
||||
r.trackPopN(2)
|
||||
pos := r.trackPeekN(1)
|
||||
r.trackPushNeg2(r.trackPeek(), 1) // Save old mark, note that we pushed a new mark
|
||||
r.stackPush(pos) // Make new mark
|
||||
r.textto(pos) // Recall position
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.trackPushNeg2(r.trackPeek(), 1) // Save old mark, note that we pushed a new mark
|
||||
r.stackPush(pos) // Make new mark
|
||||
r.textto(pos) // Recall position
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
continue
|
||||
|
||||
case syntax.Lazybranchmark | syntax.Back2:
|
||||
@@ -413,9 +493,11 @@ func executeDefault(r *Runner) error {
|
||||
r.trackPushNeg2(mark, count) // Save old mark, count
|
||||
r.advance(2) // Straight
|
||||
} else { // Nonempty match -> count+loop now
|
||||
r.trackPush1(mark) // remember mark
|
||||
r.stackPush2(r.textPos(), count+1) // Make new mark, incr count
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.trackPush1(mark) // remember mark
|
||||
r.stackPush2(r.textPos(), count+1) // Make new mark, incr count
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
|
||||
@@ -452,9 +534,11 @@ func executeDefault(r *Runner) error {
|
||||
count := r.stackPeekN(1)
|
||||
|
||||
if count < 0 { // Negative count -> loop now
|
||||
r.trackPushNeg1(mark) // Save old mark
|
||||
r.stackPush2(r.textPos(), count+1) // Make new mark, incr count
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.trackPushNeg1(mark) // Save old mark
|
||||
r.stackPush2(r.textPos(), count+1) // Make new mark, incr count
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
} else { // Nonneg count -> straight now
|
||||
r.trackPush3(mark, count, r.textPos()) // Save mark, count, position
|
||||
r.advance(2) // Straight
|
||||
@@ -472,10 +556,12 @@ func executeDefault(r *Runner) error {
|
||||
textpos := r.trackPeekN(2)
|
||||
|
||||
if r.trackPeekN(1) < r.operand(1) && textpos != mark { // Under limit and not empty match -> loop
|
||||
r.textto(textpos) // Recall position
|
||||
r.stackPush2(textpos, r.trackPeekN(1)+1) // Make new mark, incr count
|
||||
r.trackPushNeg1(mark) // Save old mark
|
||||
r.goTo(r.operand(0)) // Loop
|
||||
r.textto(textpos) // Recall position
|
||||
r.stackPush2(textpos, r.trackPeekN(1)+1) // Make new mark, incr count
|
||||
r.trackPushNeg1(mark) // Save old mark
|
||||
if err := r.goTo(r.operand(0)); err != nil { // Loop
|
||||
return err
|
||||
}
|
||||
continue
|
||||
} else { // Max loops or empty match -> backtrack
|
||||
r.stackPush2(r.trackPeek(), r.trackPeekN(1)) // Recall old mark, count
|
||||
@@ -648,6 +734,13 @@ func executeDefault(r *Runner) error {
|
||||
r.advance(1)
|
||||
continue
|
||||
|
||||
case syntax.Grapheme:
|
||||
if !r.TryMatchGrapheme(r.rightToLeft) {
|
||||
break
|
||||
}
|
||||
r.advance(0)
|
||||
continue
|
||||
|
||||
case syntax.Ref:
|
||||
|
||||
capnum := r.operand(0)
|
||||
@@ -935,18 +1028,21 @@ func executeDefault(r *Runner) error {
|
||||
;
|
||||
|
||||
// "break Backward" comes here:
|
||||
r.backtrack()
|
||||
if err := r.backtrack(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// increase the size of stack and track storage
|
||||
func (r *Runner) ensureStorage() {
|
||||
func (r *Runner) ensureStorage() error {
|
||||
if r.Runstackpos < r.runtrackcount*4 {
|
||||
doubleIntSlice(&r.runstack, &r.Runstackpos)
|
||||
}
|
||||
if r.Runtrackpos < r.runtrackcount*4 {
|
||||
doubleIntSlice(&r.runtrack, &r.Runtrackpos)
|
||||
if r.Runtrackpos < r.runtrackcount*4 && !r.growTrack() {
|
||||
return ErrBacktrackingStackLimit
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) ensureStack(plus int) {
|
||||
@@ -990,14 +1086,17 @@ func (r *Runner) advance(i int) {
|
||||
r.setOperator(r.code.Codes[r.codepos])
|
||||
}
|
||||
|
||||
func (r *Runner) goTo(newpos int) {
|
||||
func (r *Runner) goTo(newpos int) error {
|
||||
// when branching backward or in place, ensure storage
|
||||
if newpos <= r.codepos {
|
||||
r.ensureStorage()
|
||||
if err := r.ensureStorage(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
r.setOperator(r.code.Codes[newpos])
|
||||
r.codepos = newpos
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) textto(newpos int) {
|
||||
@@ -1016,11 +1115,47 @@ func (r *Runner) textPos() int {
|
||||
return r.Runtextpos
|
||||
}
|
||||
|
||||
// TryMatchGrapheme consumes one Unicode extended grapheme cluster in the
|
||||
// runner's current direction. It is exported for regexp2cg-generated engines.
|
||||
func (r *Runner) TryMatchGrapheme(rightToLeft bool) bool {
|
||||
var boundary int
|
||||
if rightToLeft {
|
||||
boundary = syntax.PreviousGraphemeClusterBoundary(r.Runtext, r.Runtextpos)
|
||||
} else {
|
||||
boundary = syntax.NextGraphemeClusterBoundary(r.Runtext, r.Runtextpos)
|
||||
}
|
||||
if boundary < 0 {
|
||||
return false
|
||||
}
|
||||
r.Runtextpos = boundary
|
||||
return true
|
||||
}
|
||||
|
||||
// push onto the backtracking stack
|
||||
func (r *Runner) trackpos() int {
|
||||
return len(r.runtrack) - r.Runtrackpos
|
||||
}
|
||||
|
||||
func (r *Runner) growTrack() bool {
|
||||
oldLen := len(r.runtrack)
|
||||
newLen := oldLen * 2
|
||||
if newLen == 0 {
|
||||
newLen = 1
|
||||
}
|
||||
if limit := r.re.optimizations.MaxBacktrackingStackSize; limit >= 0 && newLen > limit {
|
||||
newLen = limit
|
||||
}
|
||||
if newLen <= oldLen {
|
||||
return false
|
||||
}
|
||||
|
||||
newTrack := make([]int, newLen)
|
||||
copy(newTrack[newLen-oldLen:], r.runtrack)
|
||||
r.Runtrackpos += newLen - oldLen
|
||||
r.runtrack = newTrack
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *Runner) trackPush() {
|
||||
r.Runtrackpos--
|
||||
r.runtrack[r.Runtrackpos] = r.codepos
|
||||
@@ -1069,7 +1204,7 @@ func (r *Runner) trackPushNeg2(I1, I2 int) {
|
||||
r.runtrack[r.Runtrackpos] = -r.codepos
|
||||
}
|
||||
|
||||
func (r *Runner) backtrack() {
|
||||
func (r *Runner) backtrack() error {
|
||||
newpos := r.runtrack[r.Runtrackpos]
|
||||
r.Runtrackpos++
|
||||
|
||||
@@ -1090,10 +1225,13 @@ func (r *Runner) backtrack() {
|
||||
|
||||
// When branching backward, ensure storage
|
||||
if newpos < r.codepos {
|
||||
r.ensureStorage()
|
||||
if err := r.ensureStorage(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
r.codepos = newpos
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) setOperator(op int) {
|
||||
@@ -1397,7 +1535,8 @@ func shouldUseFindFirstCharOptimized(r *Runner) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
switch r.code.FindOptimizations.FindMode {
|
||||
opts := r.code.FindOptimizations
|
||||
switch opts.FindMode {
|
||||
case syntax.TrailingAnchor_FixedLength_LeftToRight_End,
|
||||
syntax.LeadingString_OrdinalIgnoreCase_LeftToRight,
|
||||
syntax.LeadingStrings_LeftToRight,
|
||||
@@ -1408,6 +1547,13 @@ func shouldUseFindFirstCharOptimized(r *Runner) bool {
|
||||
syntax.LiteralAfterLoop_LeftToRight,
|
||||
syntax.RequiredLandmarkChain_LeftToRight:
|
||||
return true
|
||||
case syntax.LeadingSet_LeftToRight:
|
||||
// General Unicode sets already have a direct fallback loop below.
|
||||
// Large enumerated sets are also faster through the set's ASCII bitmap
|
||||
// than through the linear IndexOfAny helper.
|
||||
return len(opts.FixedDistanceSets) > 0 &&
|
||||
((len(opts.FixedDistanceSets[0].Chars) > 0 && len(opts.FixedDistanceSets[0].Chars) <= 5) ||
|
||||
opts.FixedDistanceSets[0].Range != nil)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -1429,10 +1575,10 @@ func findFirstCharOptimized(r *Runner) (handled bool, found bool) {
|
||||
case syntax.LeadingString_OrdinalIgnoreCase_LeftToRight:
|
||||
return true, findLeadingStringLeftToRight(r, []rune(opts.LeadingPrefix), true)
|
||||
case syntax.LeadingStrings_LeftToRight:
|
||||
return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, false)
|
||||
return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, opts.LeadingPrefixFirstRunes, false)
|
||||
case syntax.LeadingStrings_OrdinalIgnoreCase_LeftToRight:
|
||||
return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, true)
|
||||
case syntax.FixedDistanceSets_LeftToRight:
|
||||
return true, findLeadingStringsLeftToRight(r, opts.LeadingPrefixesRunes, opts.LeadingPrefixFirstRunes, true)
|
||||
case syntax.LeadingSet_LeftToRight, syntax.FixedDistanceSets_LeftToRight:
|
||||
return true, findFixedDistanceSetsLeftToRight(r, opts.FixedDistanceSets)
|
||||
case syntax.FixedDistanceChar_LeftToRight:
|
||||
return true, findFixedDistanceCharLeftToRight(r, opts.FixedDistanceLiteral.C, opts.FixedDistanceLiteral.Distance)
|
||||
@@ -1487,29 +1633,69 @@ func findLeadingStringLeftToRight(r *Runner, prefix []rune, ignoreCase bool) boo
|
||||
return true
|
||||
}
|
||||
|
||||
func findLeadingStringsLeftToRight(r *Runner, prefixes [][]rune, ignoreCase bool) bool {
|
||||
func findLeadingStringsLeftToRight(r *Runner, prefixes [][]rune, firstRunes []rune, ignoreCase bool) bool {
|
||||
if len(prefixes) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for start := r.Runtextpos; start <= latestPossibleStart(r); start++ {
|
||||
for _, prefix := range prefixes {
|
||||
if ignoreCase {
|
||||
if helpers.StartsWithIgnoreCase(r.Runtext[start:], prefix) {
|
||||
// Unicode ordinal-ignore-case matching has more possible first-rune folds
|
||||
// than a small precomputed set can safely represent. Keep its conservative
|
||||
// position-by-position scan; the common case-sensitive path skips directly
|
||||
// between possible first runes.
|
||||
if ignoreCase || len(firstRunes) == 0 {
|
||||
for start := r.Runtextpos; start <= latestPossibleStart(r); start++ {
|
||||
for _, prefix := range prefixes {
|
||||
if ignoreCase {
|
||||
if helpers.StartsWithIgnoreCase(r.Runtext[start:], prefix) {
|
||||
r.Runtextpos = start
|
||||
return true
|
||||
}
|
||||
} else if helpers.StartsWith(r.Runtext[start:], prefix) {
|
||||
r.Runtextpos = start
|
||||
return true
|
||||
}
|
||||
} else if helpers.StartsWith(r.Runtext[start:], prefix) {
|
||||
}
|
||||
}
|
||||
r.Runtextpos = r.Runtextend
|
||||
return false
|
||||
}
|
||||
|
||||
latest := min(latestPossibleStart(r), r.Runtextend-1)
|
||||
for searchAt := r.Runtextpos; searchAt <= latest; {
|
||||
offset := indexOfAnyRunes(r.Runtext[searchAt:latest+1], firstRunes)
|
||||
if offset < 0 {
|
||||
break
|
||||
}
|
||||
start := searchAt + offset
|
||||
first := r.Runtext[start]
|
||||
for _, prefix := range prefixes {
|
||||
if len(prefix) > 0 && prefix[0] == first && helpers.StartsWith(r.Runtext[start:], prefix) {
|
||||
r.Runtextpos = start
|
||||
return true
|
||||
}
|
||||
}
|
||||
searchAt = start + 1
|
||||
}
|
||||
|
||||
r.Runtextpos = r.Runtextend
|
||||
return false
|
||||
}
|
||||
|
||||
func indexOfAnyRunes(input, find []rune) int {
|
||||
switch len(find) {
|
||||
case 0:
|
||||
return -1
|
||||
case 1:
|
||||
return helpers.IndexOfAny1(input, find[0])
|
||||
case 2:
|
||||
return helpers.IndexOfAny2(input, find[0], find[1])
|
||||
case 3:
|
||||
return helpers.IndexOfAny3(input, find[0], find[1], find[2])
|
||||
default:
|
||||
return helpers.IndexOfAny(input, find)
|
||||
}
|
||||
}
|
||||
|
||||
func findFixedDistanceCharLeftToRight(r *Runner, ch rune, distance int) bool {
|
||||
searchStart := r.Runtextpos + distance
|
||||
for searchStart < r.Runtextend {
|
||||
@@ -1850,6 +2036,9 @@ func (r *Runner) initMatch(textInfo *matchText) {
|
||||
if tracksize < 64 {
|
||||
tracksize = 64
|
||||
}
|
||||
if limit := r.re.optimizations.MaxBacktrackingStackSize; limit >= 0 && tracksize > limit {
|
||||
tracksize = limit
|
||||
}
|
||||
if stacksize < 32 {
|
||||
stacksize = 32
|
||||
}
|
||||
@@ -1882,9 +2071,7 @@ func (r *Runner) tidyMatch(quick bool) *Match {
|
||||
m.textpos = r.Runtextpos
|
||||
if m.matchcount[0] > 0 {
|
||||
interval := m.matches[0]
|
||||
// bytes indices aren't used so just use fast path
|
||||
m.RuneIndex = interval[0]
|
||||
m.RuneLength = interval[1]
|
||||
setCaptureFields(&m.Capture, interval[0], interval[1])
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -2056,36 +2243,6 @@ func (r *Runner) initTrackCount() {
|
||||
}
|
||||
}
|
||||
|
||||
// decodeString converts s to []rune using a shared size-classed buffer pool when
|
||||
// allowed by the regexp optimization settings. Pooled slices must be returned
|
||||
// after the runner is done with them.
|
||||
func (r *Runner) decodeString(s string) ([]rune, *[]rune) {
|
||||
buf, pooled := pooledRuneBuffers.get(len(s), r.re.optimizations.MaxCachedRuneBufferLength)
|
||||
n := 0
|
||||
for _, ch := range s {
|
||||
buf[n] = ch
|
||||
n++
|
||||
}
|
||||
return buf[:n], pooled
|
||||
}
|
||||
|
||||
func (r *Runner) decodeStringWithStart(s string, startAt int) (runes []rune, runeStart int, pooled *[]rune) {
|
||||
buf, pooled := pooledRuneBuffers.get(len(s), r.re.optimizations.MaxCachedRuneBufferLength)
|
||||
n := 0
|
||||
runeStart = -1
|
||||
for strIdx, ch := range s {
|
||||
if startAt >= 0 && strIdx == startAt {
|
||||
runeStart = n
|
||||
}
|
||||
buf[n] = ch
|
||||
n++
|
||||
}
|
||||
if startAt >= 0 && startAt == len(s) {
|
||||
runeStart = n
|
||||
}
|
||||
return buf[:n], runeStart, pooled
|
||||
}
|
||||
|
||||
// getRunner returns a runner to use for matching re.
|
||||
func (re *Regexp) getRunner() *Runner {
|
||||
if re.runnerPool == nil {
|
||||
@@ -2097,6 +2254,7 @@ func (re *Regexp) getRunner() *Runner {
|
||||
// putRunner returns a runner to the re's pool cache.
|
||||
func (re *Regexp) putRunner(r *Runner) {
|
||||
r.Runtext = nil
|
||||
r.code = re.code
|
||||
if r.runmatch != nil {
|
||||
r.runmatch.text = nil
|
||||
}
|
||||
@@ -2128,6 +2286,12 @@ func (r *Runner) StackPop() int {
|
||||
return val
|
||||
}
|
||||
|
||||
// StackDepth returns the number of integer slots currently used by the
|
||||
// generated engine's backtracking stack.
|
||||
func (r *Runner) StackDepth() int {
|
||||
return len(r.runstack) - r.Runstackpos
|
||||
}
|
||||
|
||||
func (r *Runner) StackPush(val int) {
|
||||
// check if we need to size up stack
|
||||
r.ensureStack(1)
|
||||
|
||||
+7
-11
@@ -33,21 +33,20 @@ func (re *Regexp) Split(input string, count int) ([]string, error) {
|
||||
// iterate through the matches
|
||||
priorIndex := 0
|
||||
var retVal []string
|
||||
var txt []rune
|
||||
matched := false
|
||||
|
||||
m, err := re.FindStringMatch(input)
|
||||
|
||||
for ; m != nil && count > 0; m, err = re.FindNextMatch(m) {
|
||||
txt = m.text.runes
|
||||
// if we have an m, we don't have an err
|
||||
// append our match
|
||||
retVal = append(retVal, string(txt[priorIndex:m.RuneIndex]))
|
||||
matched = true
|
||||
start, end := matchInputSpan(m)
|
||||
retVal = append(retVal, input[priorIndex:start])
|
||||
// append any capture groups, skipping group 0
|
||||
gs := m.Groups()
|
||||
for i := 1; i < len(gs); i++ {
|
||||
retVal = append(retVal, gs[i].String())
|
||||
}
|
||||
priorIndex = m.RuneIndex + m.RuneLength
|
||||
priorIndex = end
|
||||
count--
|
||||
}
|
||||
|
||||
@@ -55,13 +54,10 @@ func (re *Regexp) Split(input string, count int) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if txt == nil {
|
||||
// we never matched, return the original string
|
||||
if !matched {
|
||||
return []string{input}, nil
|
||||
}
|
||||
|
||||
// append our remainder
|
||||
retVal = append(retVal, string(txt[priorIndex:]))
|
||||
|
||||
retVal = append(retVal, input[priorIndex:])
|
||||
return retVal, nil
|
||||
}
|
||||
|
||||
+90
@@ -39,6 +39,15 @@ func newStringPrefixFilter(code *syntax.Code) StringPrefixFilter {
|
||||
return stringIndexPrefixesFilter(opts.LeadingPrefixes, false, minRequiredLength)
|
||||
case syntax.LeadingStrings_OrdinalIgnoreCase_LeftToRight:
|
||||
return stringIndexPrefixesFilter(opts.LeadingPrefixes, true, minRequiredLength)
|
||||
case syntax.LeadingSet_LeftToRight:
|
||||
if len(opts.FixedDistanceSets) == 0 {
|
||||
return nil
|
||||
}
|
||||
set := opts.FixedDistanceSets[0]
|
||||
if set.Range == nil && (len(set.Chars) == 0 || len(set.Chars) > 5) {
|
||||
return nil
|
||||
}
|
||||
return stringFixedDistanceSetFilter(set, minRequiredLength)
|
||||
case syntax.FixedDistanceChar_LeftToRight:
|
||||
return stringFixedDistanceCharFilter(opts.FixedDistanceLiteral.C, opts.FixedDistanceLiteral.Distance, minRequiredLength)
|
||||
case syntax.FixedDistanceString_LeftToRight:
|
||||
@@ -50,6 +59,87 @@ func newStringPrefixFilter(code *syntax.Code) StringPrefixFilter {
|
||||
}
|
||||
}
|
||||
|
||||
type asciiSetStringScanner struct {
|
||||
chars string
|
||||
first byte
|
||||
last byte
|
||||
useRange bool
|
||||
distance int
|
||||
}
|
||||
|
||||
func newASCIISetStringScanner(set syntax.FixedDistanceSet) (asciiSetStringScanner, bool) {
|
||||
if set.Negated || set.Distance < 0 {
|
||||
return asciiSetStringScanner{}, false
|
||||
}
|
||||
if set.Range != nil {
|
||||
if set.Range.First < 0 || set.Range.Last > utf8.RuneSelf-1 {
|
||||
return asciiSetStringScanner{}, false
|
||||
}
|
||||
return asciiSetStringScanner{
|
||||
first: byte(set.Range.First),
|
||||
last: byte(set.Range.Last),
|
||||
useRange: true,
|
||||
distance: set.Distance,
|
||||
}, true
|
||||
}
|
||||
if len(set.Chars) == 0 {
|
||||
return asciiSetStringScanner{}, false
|
||||
}
|
||||
chars := make([]byte, len(set.Chars))
|
||||
for i, ch := range set.Chars {
|
||||
if ch < 0 || ch > utf8.RuneSelf-1 {
|
||||
return asciiSetStringScanner{}, false
|
||||
}
|
||||
chars[i] = byte(ch)
|
||||
}
|
||||
return asciiSetStringScanner{chars: string(chars), distance: set.Distance}, true
|
||||
}
|
||||
|
||||
func stringFixedDistanceSetFilter(set syntax.FixedDistanceSet, minRequiredLength int) StringPrefixFilter {
|
||||
scanner, ok := newASCIISetStringScanner(set)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
return func(input string, startAt int) (candidateByteIndex int, ok bool) {
|
||||
if !hasMinRequiredBytes(input, startAt, minRequiredLength) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
for searchAt := startAt; searchAt < len(input); {
|
||||
offset := scanner.index(input[searchAt:])
|
||||
if offset < 0 {
|
||||
return 0, false
|
||||
}
|
||||
setByteIndex := searchAt + offset
|
||||
candidateByteIndex, valid := stringFixedDistanceCandidateStart(input, startAt, setByteIndex, scanner.distance)
|
||||
if valid && hasMinRequiredBytes(input, candidateByteIndex, minRequiredLength) {
|
||||
return candidateByteIndex, true
|
||||
}
|
||||
if valid {
|
||||
return 0, false
|
||||
}
|
||||
searchAt = setByteIndex + 1
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func (s asciiSetStringScanner) index(input string) int {
|
||||
if !s.useRange {
|
||||
if len(s.chars) == 1 {
|
||||
return strings.IndexByte(input, s.chars[0])
|
||||
}
|
||||
return strings.IndexAny(input, s.chars)
|
||||
}
|
||||
for i := 0; i < len(input); i++ {
|
||||
if input[i] >= s.first && input[i] <= s.last {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func stringIndexPrefixFilter(prefix string, ignoreCase bool, minRequiredLength int) StringPrefixFilter {
|
||||
if prefix == "" {
|
||||
return nil
|
||||
|
||||
+69
-3
@@ -91,6 +91,11 @@ const (
|
||||
Setloopatomic InstOp = 45
|
||||
// Updates the bumpalong position to the current position.
|
||||
UpdateBumpalong InstOp = 46
|
||||
// Matches one Unicode extended grapheme cluster (\X).
|
||||
Grapheme InstOp = 47
|
||||
// Selects and consumes the next character using a disjoint branch table.
|
||||
// Operand 0 is an index into Code.Dispatches.
|
||||
Dispatch InstOp = 48
|
||||
|
||||
// Modifiers for alternate modes
|
||||
|
||||
@@ -105,6 +110,7 @@ type Code struct {
|
||||
Codes []int // the code
|
||||
Strings [][]rune // string table
|
||||
Sets []*CharSet //character set table
|
||||
Dispatches []DispatchTable // shared match tables for deterministic alternations
|
||||
TrackCount int // how many instructions use backtracking
|
||||
Caps map[int]int // mapping of user group numbers -> impl group slots
|
||||
Capsize int // number of impl group slots
|
||||
@@ -113,6 +119,56 @@ type Code struct {
|
||||
Anchors AnchorLoc // the set of zero-length start anchors (RegexFCD.Bol, etc)
|
||||
RightToLeft bool // true if right to left
|
||||
FindOptimizations *FindOptimizations // analyzed candidate search strategy
|
||||
QuickCodes []int // bool-only code with unobservable captures removed
|
||||
QuickDispatches []DispatchTable // lightweight tables targeting QuickCodes
|
||||
CaptureSlotInUse []bool // capture slots observable by the pattern itself during quick matches
|
||||
// LeftContextRunes is how many runes before a candidate start matching may
|
||||
// inspect. 0 means none, 1 means a single previous rune (or slack so ^/\A
|
||||
// do not see the candidate as the origin), and -1 means do not slice
|
||||
// (lookbehind or \G).
|
||||
LeftContextRunes int
|
||||
}
|
||||
|
||||
// DispatchTable maps disjoint character sets to branch indices. Larger tables
|
||||
// use a direct ASCII lookup, small tables use compact masks, and non-ASCII
|
||||
// runes inspect the full sets.
|
||||
type DispatchTable struct {
|
||||
Sets []int
|
||||
Branches []int
|
||||
ASCII *[128]uint16 // entry index + 1; zero means no branch
|
||||
ASCIIMasks []uint64 // two membership words per branch when ASCII is nil
|
||||
}
|
||||
|
||||
// captureSlotsInUse returns the capture slots whose values can affect matching.
|
||||
// Group 0 is always retained as the success marker. Ordinary captures that are
|
||||
// never referenced by the pattern may be omitted by bool-only matching APIs.
|
||||
func captureSlotsInUse(codes []int, capsize int) []bool {
|
||||
inUse := make([]bool, capsize)
|
||||
if capsize > 0 {
|
||||
inUse[0] = true
|
||||
}
|
||||
for pos := 0; pos < len(codes); {
|
||||
op := InstOp(codes[pos]) & Mask
|
||||
switch op {
|
||||
case Ref, Testref:
|
||||
capnum := codes[pos+1]
|
||||
if capnum >= 0 && capnum < len(inUse) {
|
||||
inUse[capnum] = true
|
||||
}
|
||||
case Capturemark:
|
||||
// Balancing groups both observe and mutate capture state. Keep both
|
||||
// sides live even if no later backreference refers to them.
|
||||
if codes[pos+2] != -1 {
|
||||
for _, capnum := range codes[pos+1 : pos+3] {
|
||||
if capnum >= 0 && capnum < len(inUse) {
|
||||
inUse[capnum] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pos += opcodeSize(op)
|
||||
}
|
||||
return inUse
|
||||
}
|
||||
|
||||
// PrepareCharSetASCIIBitmaps builds bounded ASCII lookup tables for compiled
|
||||
@@ -156,11 +212,11 @@ func opcodeSize(op InstOp) int {
|
||||
|
||||
switch op {
|
||||
case Nothing, Bol, Eol, Boundary, Nonboundary, ECMABoundary, NonECMABoundary, Beginning, Start, EndZ,
|
||||
End, Nullmark, Setmark, Getmark, Setjump, Backjump, Forejump, Stop, UpdateBumpalong:
|
||||
End, Nullmark, Setmark, Getmark, Setjump, Backjump, Forejump, Stop, UpdateBumpalong, Grapheme:
|
||||
return 1
|
||||
|
||||
case One, Notone, Multi, Ref, Testref, Goto, Nullcount, Setcount, Lazybranch, Branchmark, Lazybranchmark,
|
||||
Prune, Set:
|
||||
Prune, Set, Dispatch:
|
||||
return 2
|
||||
|
||||
case Capturemark, Branchcount, Lazybranchcount, Onerep, Notonerep, Oneloop, Notoneloop, Onelazy, Notonelazy,
|
||||
@@ -187,7 +243,7 @@ var codeStr = []string{
|
||||
"Prune", "Stop",
|
||||
"ECMABoundary", "NonECMABoundary",
|
||||
"Oneloopatomic", "Notoneloopatomic", "Setloopatomic",
|
||||
"Bumpalong",
|
||||
"Bumpalong", "Grapheme", "Dispatch",
|
||||
}
|
||||
|
||||
func operatorDescription(op InstOp) string {
|
||||
@@ -234,6 +290,16 @@ func (c *Code) OpcodeDescription(offset int) string {
|
||||
buf.WriteString("Set = ")
|
||||
buf.WriteString(c.Sets[c.Codes[offset+1]].String())
|
||||
|
||||
case Dispatch:
|
||||
tableIndex := c.Codes[offset+1]
|
||||
fmt.Fprintf(buf, "Table = %d", tableIndex)
|
||||
if tableIndex >= 0 && tableIndex < len(c.Dispatches) {
|
||||
table := &c.Dispatches[tableIndex]
|
||||
for i, setIndex := range table.Sets {
|
||||
fmt.Fprintf(buf, ", %s -> %d", c.Sets[setIndex].String(), table.Branches[i])
|
||||
}
|
||||
}
|
||||
|
||||
case Multi:
|
||||
fmt.Fprintf(buf, "String = %s", string(c.Strings[c.Codes[offset+1]]))
|
||||
|
||||
|
||||
+294
@@ -0,0 +1,294 @@
|
||||
package syntax
|
||||
|
||||
import "unicode"
|
||||
|
||||
type graphemeBreakClass uint8
|
||||
|
||||
const (
|
||||
graphemeOther graphemeBreakClass = iota
|
||||
graphemeCR
|
||||
graphemeLF
|
||||
graphemeControl
|
||||
graphemeExtend
|
||||
graphemeZWJ
|
||||
graphemeRegionalIndicator
|
||||
graphemePrepend
|
||||
graphemeSpacingMark
|
||||
graphemeL
|
||||
graphemeV
|
||||
graphemeT
|
||||
graphemeLV
|
||||
graphemeLVT
|
||||
)
|
||||
|
||||
type indicConjunctBreakClass uint8
|
||||
|
||||
const (
|
||||
indicNone indicConjunctBreakClass = iota
|
||||
indicConsonant
|
||||
indicExtend
|
||||
indicLinker
|
||||
)
|
||||
|
||||
type graphemeProperties struct {
|
||||
breakClass graphemeBreakClass
|
||||
indicClass indicConjunctBreakClass
|
||||
extendedPictographic bool
|
||||
}
|
||||
|
||||
// NextGraphemeClusterBoundary returns the first extended-grapheme boundary
|
||||
// after start, or -1 when start is not a valid input position. It implements
|
||||
// the Unicode 17 rules from UAX #29 with a small state machine, avoiding the
|
||||
// backtracking and repeated character-class probes of an equivalent regexp.
|
||||
func NextGraphemeClusterBoundary(text []rune, start int) int {
|
||||
if start < 0 || start >= len(text) {
|
||||
return -1
|
||||
}
|
||||
|
||||
previous := graphemePropertiesFor(text[start])
|
||||
state := graphemeForwardState{}
|
||||
state.consume(previous)
|
||||
|
||||
for pos := start + 1; pos < len(text); pos++ {
|
||||
current := graphemePropertiesFor(text[pos])
|
||||
if isGraphemeBoundaryForward(previous.breakClass, current, state) {
|
||||
return pos
|
||||
}
|
||||
state.consume(current)
|
||||
previous = current
|
||||
}
|
||||
return len(text)
|
||||
}
|
||||
|
||||
// PreviousGraphemeClusterBoundary returns the first extended-grapheme boundary
|
||||
// before end, or -1 when end is not a valid input position.
|
||||
func PreviousGraphemeClusterBoundary(text []rune, end int) int {
|
||||
if end <= 0 || end > len(text) {
|
||||
return -1
|
||||
}
|
||||
for pos := end - 1; pos > 0; pos-- {
|
||||
if isGraphemeBoundary(text, pos) {
|
||||
return pos
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type graphemeForwardState struct {
|
||||
regionalIndicatorCount int
|
||||
extendedPictographicRun bool
|
||||
zwjAfterPictographic bool
|
||||
indicConsonantRun bool
|
||||
indicLinkerSeen bool
|
||||
}
|
||||
|
||||
func (s *graphemeForwardState) consume(properties graphemeProperties) {
|
||||
if properties.breakClass == graphemeRegionalIndicator {
|
||||
s.regionalIndicatorCount++
|
||||
} else {
|
||||
s.regionalIndicatorCount = 0
|
||||
}
|
||||
|
||||
if properties.breakClass == graphemeZWJ {
|
||||
s.zwjAfterPictographic = s.extendedPictographicRun
|
||||
} else {
|
||||
s.zwjAfterPictographic = false
|
||||
}
|
||||
if properties.extendedPictographic {
|
||||
s.extendedPictographicRun = true
|
||||
} else if properties.breakClass != graphemeExtend {
|
||||
s.extendedPictographicRun = false
|
||||
}
|
||||
|
||||
switch properties.indicClass {
|
||||
case indicConsonant:
|
||||
s.indicConsonantRun = true
|
||||
s.indicLinkerSeen = false
|
||||
case indicExtend:
|
||||
// Extend preserves a preceding consonant/linker run.
|
||||
case indicLinker:
|
||||
if s.indicConsonantRun {
|
||||
s.indicLinkerSeen = true
|
||||
}
|
||||
default:
|
||||
s.indicConsonantRun = false
|
||||
s.indicLinkerSeen = false
|
||||
}
|
||||
}
|
||||
|
||||
func isGraphemeBoundaryForward(previous graphemeBreakClass, current graphemeProperties, state graphemeForwardState) bool {
|
||||
if previous == graphemeCR && current.breakClass == graphemeLF { // GB3
|
||||
return false
|
||||
}
|
||||
if isGraphemeControl(previous) || isGraphemeControl(current.breakClass) { // GB4, GB5
|
||||
return true
|
||||
}
|
||||
if hangulNoBreak(previous, current.breakClass) { // GB6, GB7, GB8
|
||||
return false
|
||||
}
|
||||
if current.breakClass == graphemeExtend || current.breakClass == graphemeZWJ || current.breakClass == graphemeSpacingMark { // GB9, GB9a
|
||||
return false
|
||||
}
|
||||
if previous == graphemePrepend { // GB9b
|
||||
return false
|
||||
}
|
||||
if current.indicClass == indicConsonant && state.indicConsonantRun && state.indicLinkerSeen { // GB9c
|
||||
return false
|
||||
}
|
||||
if current.extendedPictographic && state.zwjAfterPictographic { // GB11
|
||||
return false
|
||||
}
|
||||
if previous == graphemeRegionalIndicator && current.breakClass == graphemeRegionalIndicator && state.regionalIndicatorCount%2 == 1 { // GB12, GB13
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isGraphemeBoundary(text []rune, pos int) bool {
|
||||
previous := graphemePropertiesFor(text[pos-1])
|
||||
current := graphemePropertiesFor(text[pos])
|
||||
if previous.breakClass == graphemeCR && current.breakClass == graphemeLF { // GB3
|
||||
return false
|
||||
}
|
||||
if isGraphemeControl(previous.breakClass) || isGraphemeControl(current.breakClass) { // GB4, GB5
|
||||
return true
|
||||
}
|
||||
if hangulNoBreak(previous.breakClass, current.breakClass) { // GB6, GB7, GB8
|
||||
return false
|
||||
}
|
||||
if current.breakClass == graphemeExtend || current.breakClass == graphemeZWJ || current.breakClass == graphemeSpacingMark { // GB9, GB9a
|
||||
return false
|
||||
}
|
||||
if previous.breakClass == graphemePrepend { // GB9b
|
||||
return false
|
||||
}
|
||||
if current.indicClass == indicConsonant && hasIndicConjunctBefore(text, pos) { // GB9c
|
||||
return false
|
||||
}
|
||||
if current.extendedPictographic && hasExtendedPictographicZWJBefore(text, pos) { // GB11
|
||||
return false
|
||||
}
|
||||
if previous.breakClass == graphemeRegionalIndicator && current.breakClass == graphemeRegionalIndicator { // GB12, GB13
|
||||
count := 0
|
||||
for i := pos - 1; i >= 0 && graphemeClass(text[i]) == graphemeRegionalIndicator; i-- {
|
||||
count++
|
||||
}
|
||||
return count%2 == 0
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func hasIndicConjunctBefore(text []rune, pos int) bool {
|
||||
linkerSeen := false
|
||||
for i := pos - 1; i >= 0; i-- {
|
||||
switch indicClass(text[i]) {
|
||||
case indicExtend:
|
||||
continue
|
||||
case indicLinker:
|
||||
linkerSeen = true
|
||||
continue
|
||||
case indicConsonant:
|
||||
return linkerSeen
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasExtendedPictographicZWJBefore(text []rune, pos int) bool {
|
||||
i := pos - 1
|
||||
if i < 0 || graphemeClass(text[i]) != graphemeZWJ {
|
||||
return false
|
||||
}
|
||||
for i--; i >= 0 && graphemeClass(text[i]) == graphemeExtend; i-- {
|
||||
}
|
||||
return i >= 0 && unicode.Is(unicodeAliasExtended_Pictographic, text[i])
|
||||
}
|
||||
|
||||
func isGraphemeControl(class graphemeBreakClass) bool {
|
||||
return class == graphemeCR || class == graphemeLF || class == graphemeControl
|
||||
}
|
||||
|
||||
func hangulNoBreak(previous, current graphemeBreakClass) bool {
|
||||
return previous == graphemeL && (current == graphemeL || current == graphemeV || current == graphemeLV || current == graphemeLVT) ||
|
||||
(previous == graphemeLV || previous == graphemeV) && (current == graphemeV || current == graphemeT) ||
|
||||
(previous == graphemeLVT || previous == graphemeT) && current == graphemeT
|
||||
}
|
||||
|
||||
func indicClass(ch rune) indicConjunctBreakClass {
|
||||
if ch < 0x300 {
|
||||
return indicNone
|
||||
}
|
||||
if unicode.Is(unicodeAliasIndic_Conjunct_Break_Consonant, ch) {
|
||||
return indicConsonant
|
||||
}
|
||||
if unicode.Is(unicodeAliasIndic_Conjunct_Break_Linker, ch) {
|
||||
return indicLinker
|
||||
}
|
||||
if unicode.Is(unicodeAliasIndic_Conjunct_Break_Extend, ch) {
|
||||
return indicExtend
|
||||
}
|
||||
return indicNone
|
||||
}
|
||||
|
||||
func graphemePropertiesFor(ch rune) graphemeProperties {
|
||||
if ch <= unicode.MaxASCII {
|
||||
return graphemeProperties{breakClass: graphemeClass(ch)}
|
||||
}
|
||||
return graphemeProperties{
|
||||
breakClass: graphemeClass(ch),
|
||||
indicClass: indicClass(ch),
|
||||
extendedPictographic: ch >= 0xA9 && unicode.Is(unicodeAliasExtended_Pictographic, ch),
|
||||
}
|
||||
}
|
||||
|
||||
func graphemeClass(ch rune) graphemeBreakClass {
|
||||
if ch <= unicode.MaxASCII {
|
||||
switch ch {
|
||||
case '\r':
|
||||
return graphemeCR
|
||||
case '\n':
|
||||
return graphemeLF
|
||||
default:
|
||||
if ch < ' ' || ch == 0x7F {
|
||||
return graphemeControl
|
||||
}
|
||||
return graphemeOther
|
||||
}
|
||||
}
|
||||
if ch == 0x200D {
|
||||
return graphemeZWJ
|
||||
}
|
||||
if ch >= 0x1F1E6 && ch <= 0x1F1FF {
|
||||
return graphemeRegionalIndicator
|
||||
}
|
||||
if ch >= 0xAC00 && ch <= 0xD7A3 {
|
||||
if (ch-0xAC00)%28 == 0 {
|
||||
return graphemeLV
|
||||
}
|
||||
return graphemeLVT
|
||||
}
|
||||
if ch >= 0x1100 && ch <= 0x115F || ch >= 0xA960 && ch <= 0xA97C {
|
||||
return graphemeL
|
||||
}
|
||||
if ch >= 0x1160 && ch <= 0x11A7 || ch >= 0xD7B0 && ch <= 0xD7C6 {
|
||||
return graphemeV
|
||||
}
|
||||
if ch >= 0x11A8 && ch <= 0x11FF || ch >= 0xD7CB && ch <= 0xD7FB {
|
||||
return graphemeT
|
||||
}
|
||||
if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Control, ch) {
|
||||
return graphemeControl
|
||||
}
|
||||
if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Extend, ch) {
|
||||
return graphemeExtend
|
||||
}
|
||||
if unicode.Is(unicodeAliasGrapheme_Cluster_Break_Prepend, ch) {
|
||||
return graphemePrepend
|
||||
}
|
||||
if unicode.Is(unicodeAliasGrapheme_Cluster_Break_SpacingMark, ch) {
|
||||
return graphemeSpacingMark
|
||||
}
|
||||
return graphemeOther
|
||||
}
|
||||
+26
-24
@@ -21,10 +21,11 @@ type FindOptimizations struct {
|
||||
LeadingPrefixesRunes [][]rune
|
||||
//LeadingStrings *helpers.StringSearchValues
|
||||
|
||||
FixedDistanceLiteral FixedDistanceLiteral
|
||||
FixedDistanceSets []FixedDistanceSet
|
||||
LiteralAfterLoop *LiteralAfterLoop
|
||||
LandmarkChain *RequiredLandmarkChain
|
||||
FixedDistanceLiteral FixedDistanceLiteral
|
||||
FixedDistanceSets []FixedDistanceSet
|
||||
LiteralAfterLoop *LiteralAfterLoop
|
||||
LandmarkChain *RequiredLandmarkChain
|
||||
LeadingPrefixFirstRunes []rune
|
||||
}
|
||||
|
||||
type LiteralAfterLoop struct {
|
||||
@@ -460,19 +461,23 @@ func newFindOptimizationsForNode(root *RegexNode, opt ParseOptions, isLeadingPar
|
||||
|
||||
// We're now left-to-right only and looking for multiple prefixes and/or sets.
|
||||
|
||||
// If there are multiple leading strings, we can search for any of them.
|
||||
// this works in the interpreter, but we avoid it due to additional cost during construction
|
||||
|
||||
// Multiple leading strings let the finder jump between candidate prefixes.
|
||||
// Case-sensitive prefixes are cheap to collect and help the interpreter.
|
||||
// Case-insensitive prefix explosion is still limited to code generation.
|
||||
if prefixes := findPrefixes(root, false); len(prefixes) > 1 {
|
||||
f.LeadingPrefixes = prefixes
|
||||
f.LeadingPrefixesRunes = toRunePrefixes(prefixes)
|
||||
f.LeadingPrefixFirstRunes = leadingPrefixFirstRunes(f.LeadingPrefixesRunes)
|
||||
f.FindMode = LeadingStrings_LeftToRight
|
||||
return f
|
||||
}
|
||||
if !interpreter {
|
||||
ciPrefixes := findPrefixes(root, true)
|
||||
if len(ciPrefixes) > 1 {
|
||||
f.LeadingPrefixes = ciPrefixes
|
||||
f.LeadingPrefixesRunes = toRunePrefixes(ciPrefixes)
|
||||
f.LeadingPrefixFirstRunes = leadingPrefixFirstRunes(f.LeadingPrefixesRunes)
|
||||
f.FindMode = LeadingStrings_OrdinalIgnoreCase_LeftToRight
|
||||
/*SYSTEM_TEXT_REGULAREXPRESSIONS
|
||||
if usesRfoTryFind {
|
||||
f.LeadingStrings = helpers.NewSearchValues(f.LeadingPrefixes, true)
|
||||
}*/
|
||||
return f
|
||||
}
|
||||
}
|
||||
@@ -515,19 +520,6 @@ func newFindOptimizationsForNode(root *RegexNode, opt ParseOptions, isLeadingPar
|
||||
// In some searches, we may use multiple sets, so we want the subsequent ones to also be the efficiency runners-up.
|
||||
slices.SortFunc(fixedDistanceSets, compareFixedDistanceSetsByQuality)
|
||||
|
||||
// If the best fixed-distance set is composed of high-frequency characters, IndexOfAny on
|
||||
// those characters is likely to match too many positions. Prefer a case-sensitive
|
||||
// multi-prefix search when one is available.
|
||||
if !interpreter && !mayContainCaseInsensitiveMatching(root) && hasHighFrequencyChars(fixedDistanceSets[0]) {
|
||||
caseSensitivePrefixes := findPrefixes(root, false)
|
||||
if len(caseSensitivePrefixes) > 1 {
|
||||
f.LeadingPrefixes = caseSensitivePrefixes
|
||||
f.LeadingPrefixesRunes = toRunePrefixes(caseSensitivePrefixes)
|
||||
f.FindMode = LeadingStrings_LeftToRight
|
||||
return f
|
||||
}
|
||||
}
|
||||
|
||||
// If there is no literal after the loop, use whatever set we got.
|
||||
// If there is a literal after the loop, consider it to be better than a negated set and better than a set with many characters.
|
||||
if literalAfterLoop == nil || (len(fixedDistanceSets[0].Chars) > 0 && !fixedDistanceSets[0].Negated) {
|
||||
@@ -587,6 +579,16 @@ func toRunePrefixes(prefixes []string) [][]rune {
|
||||
return runes
|
||||
}
|
||||
|
||||
func leadingPrefixFirstRunes(prefixes [][]rune) []rune {
|
||||
first := make([]rune, 0, len(prefixes))
|
||||
for _, prefix := range prefixes {
|
||||
if len(prefix) > 0 && !slices.Contains(first, prefix[0]) {
|
||||
first = append(first, prefix[0])
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func getFindMode(rtl bool, t NodeType) FindNextStartingPositionMode {
|
||||
if rtl {
|
||||
switch t {
|
||||
|
||||
+168
-13
@@ -636,11 +636,16 @@ func (p *parser) scanRegex() (*RegexNode, error) {
|
||||
}
|
||||
|
||||
case '\\':
|
||||
quoted := !p.useOptionE() && p.charsRight() > 0 && p.rightChar(0) == 'Q'
|
||||
n, err := p.scanBackslash(false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.addUnitNode(n)
|
||||
if quoted {
|
||||
p.addQuotedUnit(n)
|
||||
} else {
|
||||
p.addUnitNode(n)
|
||||
}
|
||||
|
||||
case '^':
|
||||
if p.useOptionM() {
|
||||
@@ -702,7 +707,7 @@ func (p *parser) scanRegex() (*RegexNode, error) {
|
||||
// Handle quantifiers
|
||||
for p.unit != nil {
|
||||
var min, max int
|
||||
var lazy bool
|
||||
var lazy, possessive bool
|
||||
|
||||
switch ch {
|
||||
case '*':
|
||||
@@ -753,18 +758,19 @@ func (p *parser) scanRegex() (*RegexNode, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if p.charsRight() == 0 || p.rightChar(0) != '?' {
|
||||
lazy = false
|
||||
} else {
|
||||
if p.charsRight() > 0 && p.rightChar(0) == '?' {
|
||||
p.moveRight(1)
|
||||
lazy = true
|
||||
} else if p.charsRight() > 0 && p.rightChar(0) == '+' && !p.useOptionE() {
|
||||
p.moveRight(1)
|
||||
possessive = true
|
||||
}
|
||||
|
||||
if min > max {
|
||||
return nil, p.getErr(ErrInvalidRepeatSize)
|
||||
}
|
||||
|
||||
p.addConcatenate3(lazy, min, max)
|
||||
p.addConcatenate3(lazy, possessive, min, max)
|
||||
}
|
||||
|
||||
ContinueOuterScan:
|
||||
@@ -1280,6 +1286,37 @@ func (p *parser) scanBackslash(scanOnly bool) (*RegexNode, error) {
|
||||
}
|
||||
|
||||
switch ch := p.rightChar(0); ch {
|
||||
case 'Q':
|
||||
if p.useOptionE() {
|
||||
return p.scanBasicBackslash(scanOnly)
|
||||
}
|
||||
p.moveRight(1)
|
||||
quoted := p.scanQuoted()
|
||||
if scanOnly {
|
||||
return nil, nil
|
||||
}
|
||||
return newRegexNodeStr(NtMulti, p.options&^IgnoreCase, quoted), nil
|
||||
|
||||
case 'R':
|
||||
if p.useOptionE() || p.useRE2() {
|
||||
return p.scanBasicBackslash(scanOnly)
|
||||
}
|
||||
p.moveRight(1)
|
||||
if scanOnly {
|
||||
return nil, nil
|
||||
}
|
||||
return newUnicodeNewlineNode(p.options), nil
|
||||
|
||||
case 'X':
|
||||
if p.useOptionE() || p.useRE2() {
|
||||
return p.scanBasicBackslash(scanOnly)
|
||||
}
|
||||
p.moveRight(1)
|
||||
if scanOnly {
|
||||
return nil, nil
|
||||
}
|
||||
return newRegexNode(NtGrapheme, p.options), nil
|
||||
|
||||
case 'b', 'B', 'A', 'G', 'Z', 'z':
|
||||
p.moveRight(1)
|
||||
return newRegexNode(p.typeFromCode(ch), p.options), nil
|
||||
@@ -1354,6 +1391,58 @@ func (p *parser) scanBackslash(scanOnly bool) (*RegexNode, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// newUnicodeNewlineNode constructs \R from existing general-purpose nodes.
|
||||
// Its branches have disjoint starting sets, and the optional LF is atomic, so
|
||||
// reduction can remove the outer atomic wrapper without changing semantics.
|
||||
func newUnicodeNewlineNode(options RegexOptions) *RegexNode {
|
||||
options &^= IgnoreCase
|
||||
|
||||
lead, optional := '\r', '\n'
|
||||
if options&RightToLeft != 0 {
|
||||
// Stored concatenations are already in execution order. In RTL, consume
|
||||
// LF first and then the optional preceding CR.
|
||||
lead, optional = '\n', '\r'
|
||||
}
|
||||
|
||||
newlines := &CharSet{}
|
||||
for _, ch := range []rune{'\n', '\v', '\f', '\u0085', '\u2028', '\u2029'} {
|
||||
if ch != lead {
|
||||
newlines.addChar(ch)
|
||||
}
|
||||
}
|
||||
if '\r' != lead {
|
||||
newlines.addChar('\r')
|
||||
}
|
||||
|
||||
crlf := newRegexNode(NtConcatenate, options)
|
||||
crlf.addChild(newRegexNodeCh(NtOne, options, lead))
|
||||
crlf.addChild(newRegexNodeCh(NtOne, options, optional).makeQuantifier(false, 0, 1))
|
||||
|
||||
alternate := newRegexNode(NtAlternate, options)
|
||||
alternate.addChild(newRegexNodeSet(NtSet, options, newlines))
|
||||
alternate.addChild(crlf)
|
||||
|
||||
atomic := newRegexNode(NtAtomic, options)
|
||||
atomic.addChild(alternate)
|
||||
return atomic
|
||||
}
|
||||
|
||||
// scanQuoted scans the literal text after \Q through the next \E, or through
|
||||
// the end of the pattern when there is no terminator.
|
||||
func (p *parser) scanQuoted() []rune {
|
||||
start := p.textpos()
|
||||
var quoted []rune
|
||||
for p.charsRight() > 0 {
|
||||
if p.rightChar(0) == '\\' && p.charsRight() > 1 && p.rightChar(1) == 'E' {
|
||||
quoted = append(quoted, p.pattern[start:p.textpos()]...)
|
||||
p.moveRight(2)
|
||||
return quoted
|
||||
}
|
||||
p.moveRight(1)
|
||||
}
|
||||
return append(quoted, p.pattern[start:p.textpos()]...)
|
||||
}
|
||||
|
||||
// Scans \-style backreferences and character escapes
|
||||
func (p *parser) scanBasicBackslash(scanOnly bool) (*RegexNode, error) {
|
||||
if p.charsRight() == 0 {
|
||||
@@ -1684,6 +1773,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
|
||||
inRange := false
|
||||
firstChar := true
|
||||
closed := false
|
||||
var quoted []rune
|
||||
|
||||
var cc *CharSet
|
||||
if !scanOnly {
|
||||
@@ -1697,11 +1787,33 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
|
||||
}
|
||||
}
|
||||
|
||||
for ; p.charsRight() > 0; firstChar = false {
|
||||
for ; p.charsRight() > 0 || len(quoted) > 0; firstChar = false {
|
||||
fTranslatedChar := false
|
||||
ch = p.moveRightGetChar()
|
||||
for {
|
||||
if len(quoted) > 0 {
|
||||
ch = quoted[0]
|
||||
quoted = quoted[1:]
|
||||
fTranslatedChar = true
|
||||
break
|
||||
}
|
||||
ch = p.moveRightGetChar()
|
||||
if ch == '\\' && !p.useOptionE() && p.charsRight() > 0 && p.rightChar(0) == 'Q' {
|
||||
p.moveRight(1)
|
||||
quoted = p.scanQuoted()
|
||||
if len(quoted) == 0 {
|
||||
if p.charsRight() == 0 {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if ch == ']' {
|
||||
if !firstChar {
|
||||
if fTranslatedChar {
|
||||
// A quoted closing bracket is an ordinary class member.
|
||||
} else if !firstChar {
|
||||
closed = true
|
||||
break
|
||||
} else if p.useOptionE() {
|
||||
@@ -1712,7 +1824,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
|
||||
break
|
||||
}
|
||||
|
||||
} else if ch == '\\' && p.charsRight() > 0 {
|
||||
} else if ch == '\\' && !fTranslatedChar && p.charsRight() > 0 {
|
||||
switch ch = p.moveRightGetChar(); ch {
|
||||
case 'D', 'd':
|
||||
if !scanOnly {
|
||||
@@ -1815,7 +1927,7 @@ func (p *parser) scanCharSet(caseInsensitive, scanOnly bool) (*CharSet, error) {
|
||||
}
|
||||
fTranslatedChar = true
|
||||
}
|
||||
} else if ch == '[' {
|
||||
} else if ch == '[' && !fTranslatedChar {
|
||||
// This is code for Posix style properties - [:Ll:] or [:IsTibetan:].
|
||||
// It currently doesn't do anything other than skip the whole thing!
|
||||
if p.charsRight() > 0 && p.rightChar(0) == ':' && !inRange {
|
||||
@@ -2291,11 +2403,54 @@ func (p *parser) addConcatenate() {
|
||||
}
|
||||
|
||||
// Finish the current quantifiable (when a quantifier is found)
|
||||
func (p *parser) addConcatenate3(lazy bool, min, max int) {
|
||||
p.concatenation.addChild(p.unit.makeQuantifier(lazy, min, max))
|
||||
func (p *parser) addConcatenate3(lazy, possessive bool, min, max int) {
|
||||
node := p.unit.makeQuantifier(lazy, min, max)
|
||||
if possessive {
|
||||
atomic := newRegexNode(NtAtomic, p.options)
|
||||
atomic.addChild(node)
|
||||
node = atomic
|
||||
}
|
||||
p.concatenation.addChild(node)
|
||||
p.unit = nil
|
||||
}
|
||||
|
||||
// addQuotedUnit adds the literal characters scanned by \Q...\E. All but the
|
||||
// final character are completed immediately so a following quantifier applies
|
||||
// to the final literal character, just as if each character had been escaped.
|
||||
func (p *parser) addQuotedUnit(node *RegexNode) {
|
||||
quoted := node.Str
|
||||
if len(quoted) == 0 {
|
||||
p.restoreLastUnit()
|
||||
if p.unit == nil {
|
||||
p.unit = newRegexNode(NtEmpty, p.options)
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, ch := range quoted[:len(quoted)-1] {
|
||||
p.concatenation.addChild(newRegexNodeCh(NtOne, p.options, ch))
|
||||
}
|
||||
p.unit = newRegexNodeCh(NtOne, p.options, quoted[len(quoted)-1])
|
||||
}
|
||||
|
||||
// restoreLastUnit makes an immediately preceding literal quantifiable again
|
||||
// when an empty \Q\E appears between it and its quantifier.
|
||||
func (p *parser) restoreLastUnit() {
|
||||
if len(p.concatenation.Children) == 0 {
|
||||
return
|
||||
}
|
||||
last := len(p.concatenation.Children) - 1
|
||||
node := p.concatenation.Children[last]
|
||||
p.concatenation.Children = p.concatenation.Children[:last]
|
||||
node.Parent = nil
|
||||
if node.T == NtMulti && len(node.Str) > 1 {
|
||||
prefix := append([]rune(nil), node.Str[:len(node.Str)-1]...)
|
||||
p.concatenation.addChild(newRegexNodeStr(NtMulti, node.Options, prefix))
|
||||
p.unit = newRegexNodeCh(NtOne, p.options, node.Str[len(node.Str)-1])
|
||||
return
|
||||
}
|
||||
p.unit = node
|
||||
}
|
||||
|
||||
// Sets the current unit to a single char node
|
||||
func (p *parser) addUnitOne(ch rune) {
|
||||
p.unit = newRegexNodeCh(NtOne, p.options, ch)
|
||||
|
||||
+3
@@ -245,6 +245,9 @@ func (s *regexFcd) calculateFC(nt NodeType, node *RegexNode, CurIndex int) {
|
||||
case NtSetloop, NtSetlazy, NtSetloopatomic:
|
||||
s.pushFC(regexFc{cc: node.Set.Copy(), nullable: node.M == 0, caseInsensitive: ci})
|
||||
|
||||
case NtGrapheme:
|
||||
s.pushFC(regexFc{cc: *AnyClass(), nullable: false})
|
||||
|
||||
case NtRef:
|
||||
s.pushFC(regexFc{cc: *AnyClass(), nullable: true, caseInsensitive: false})
|
||||
|
||||
|
||||
+10
-45
@@ -123,6 +123,11 @@ func tryFindFirstCharClass(node *RegexNode, ccIn **CharSet) int {
|
||||
}
|
||||
return 0
|
||||
|
||||
case NtGrapheme:
|
||||
// Every rune can begin a grapheme, so there is no useful candidate
|
||||
// restriction to derive.
|
||||
return 0
|
||||
|
||||
// Zero-width elements. These don't contribute to the starting set, so return null to indicate a caller
|
||||
// should keep looking past them.
|
||||
case NtEmpty, NtNothing, NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary,
|
||||
@@ -532,6 +537,9 @@ func findPrefixesCore(node *RegexNode, res *[]*bytes.Buffer, ignoreCase bool) bo
|
||||
// that comprise the set. For case-insensitive, we need the set to be two ASCII letters that case fold to the same thing.
|
||||
// As with One and loops, set loops are handled the same as sets up to the min iteration limit.
|
||||
case NtSet, NtSetloop, NtSetlazy, NtSetloopatomic:
|
||||
if node.Set == nil || node.Set.IsNegated() {
|
||||
return false
|
||||
}
|
||||
|
||||
setChars := node.Set.GetSetChars(maxPrefixes)
|
||||
|
||||
@@ -895,10 +903,11 @@ func tryFindRawFixedSets(node *RegexNode, res *[]FixedDistanceSet, distance *int
|
||||
combined[fixedSet.Distance] = v
|
||||
}
|
||||
} else {
|
||||
setCopy := fixedSet.Set.Copy()
|
||||
combined[fixedSet.Distance] = struct {
|
||||
Set *CharSet
|
||||
Count int
|
||||
}{Set: fixedSet.Set, Count: 1}
|
||||
}{Set: &setCopy, Count: 1}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1034,50 +1043,6 @@ func sumFrequencies(chars []rune) float32 {
|
||||
return sum
|
||||
}
|
||||
|
||||
func hasHighFrequencyChars(set FixedDistanceSet) bool {
|
||||
if set.Negated {
|
||||
return true
|
||||
}
|
||||
|
||||
// Sets without extracted chars can't be frequency-analyzed.
|
||||
// Single-char sets use IndexOf, which is a strong filter regardless of frequency.
|
||||
if len(set.Chars) <= 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
totalFrequency := sumFrequencies(set.Chars)
|
||||
|
||||
// If the average frequency of the set's chars exceeds this threshold, the
|
||||
// characters are common enough that a multi-string search may be a better filter.
|
||||
const highFrequencyThreshold = 0.6
|
||||
return totalFrequency >= highFrequencyThreshold*float32(len(set.Chars))
|
||||
}
|
||||
|
||||
func mayContainCaseInsensitiveMatching(node *RegexNode) bool {
|
||||
if node.Options&IgnoreCase != 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
if node.Set != nil {
|
||||
chars := node.Set.GetSetChars(maxPrefixes)
|
||||
for _, ch := range chars {
|
||||
if participatesInCaseConversion(ch) &&
|
||||
slices.Contains(chars, unicode.ToLower(ch)) &&
|
||||
slices.Contains(chars, unicode.ToUpper(ch)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, child := range node.Children {
|
||||
if mayContainCaseInsensitiveMatching(child) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Percent occurrences in source text (100 * char count / total count)
|
||||
var frequency = []float32{
|
||||
0.000 /* '\x00' */, 0.000 /* '\x01' */, 0.000 /* '\x02' */, 0.000 /* '\x03' */, 0.000 /* '\x04' */, 0.000 /* '\x05' */, 0.000 /* '\x06' */, 0.000, /* '\x07' */
|
||||
|
||||
+119
-1
@@ -128,6 +128,8 @@ const (
|
||||
NtSetloopatomic NodeType = 45
|
||||
// Updates the bumpalong position to the current position.
|
||||
NtUpdateBumpalong NodeType = 46
|
||||
// Matches one Unicode extended grapheme cluster (\X).
|
||||
NtGrapheme NodeType = 47
|
||||
)
|
||||
|
||||
func newRegexNode(t NodeType, opt RegexOptions) *RegexNode {
|
||||
@@ -693,10 +695,117 @@ func (n *RegexNode) reduceAtomic() *RegexNode {
|
||||
// For everything else, try to reduce ending backtracking of the last contained expression.
|
||||
default:
|
||||
child.eliminateEndingBacktracking()
|
||||
if child.T == NtAlternate && child.hasDisjointStartingSets() && child.hasOnlyIntrinsicallyAtomicBranches() {
|
||||
return child
|
||||
}
|
||||
return atomic
|
||||
}
|
||||
}
|
||||
|
||||
// hasDisjointStartingSets reports whether every branch is non-nullable and
|
||||
// begins with a character set that cannot overlap any other branch. In that
|
||||
// case, once one branch has matched, no sibling branch could match the same
|
||||
// starting position.
|
||||
func (n *RegexNode) hasDisjointStartingSets() bool {
|
||||
_, ok := n.disjointStartingSets()
|
||||
return ok
|
||||
}
|
||||
|
||||
// disjointStartingSets returns the non-nullable first-character set for each
|
||||
// branch when all of those sets are pairwise disjoint.
|
||||
func (n *RegexNode) disjointStartingSets() ([]*CharSet, bool) {
|
||||
sets := make([]*CharSet, 0, len(n.Children))
|
||||
for _, branch := range n.Children {
|
||||
var set *CharSet
|
||||
if tryFindFirstCharClass(branch, &set) != 1 || set == nil {
|
||||
return nil, false
|
||||
}
|
||||
for _, previous := range sets {
|
||||
if set.MayOverlap(previous) {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
sets = append(sets, set)
|
||||
}
|
||||
return sets, true
|
||||
}
|
||||
|
||||
// disjointAtomicBranchSets returns the first-character sets when an
|
||||
// alternation can select a branch without leaving a backtracking choice.
|
||||
func (n *RegexNode) disjointAtomicBranchSets() ([]*CharSet, bool) {
|
||||
if n.T != NtAlternate || !n.hasOnlyIntrinsicallyAtomicBranches() {
|
||||
return nil, false
|
||||
}
|
||||
return n.disjointStartingSets()
|
||||
}
|
||||
|
||||
// dispatchCandidates returns disjoint branch sets and the leading nodes that
|
||||
// can be folded into a consuming Dispatch instruction.
|
||||
func (n *RegexNode) dispatchCandidates() ([]*CharSet, []*RegexNode, []bool, bool) {
|
||||
if len(n.Children) >= 1<<16 {
|
||||
return nil, nil, nil, false
|
||||
}
|
||||
sets, ok := n.disjointAtomicBranchSets()
|
||||
if !ok {
|
||||
return nil, nil, nil, false
|
||||
}
|
||||
|
||||
leaders := make([]*RegexNode, len(n.Children))
|
||||
complete := make([]bool, len(n.Children))
|
||||
for i := range leaders {
|
||||
branch := n.Children[i]
|
||||
leader := branch
|
||||
if leader.T == NtConcatenate && len(leader.Children) > 0 {
|
||||
leader = leader.Children[0]
|
||||
}
|
||||
switch leader.T {
|
||||
case NtOne, NtNotone, NtSet, NtMulti:
|
||||
leaders[i] = leader
|
||||
default:
|
||||
return nil, nil, nil, false
|
||||
}
|
||||
complete[i] = leader == branch && (leader.T != NtMulti || len(leader.Str) == 1)
|
||||
}
|
||||
return sets, leaders, complete, true
|
||||
}
|
||||
|
||||
func (n *RegexNode) hasOnlyIntrinsicallyAtomicBranches() bool {
|
||||
for _, branch := range n.Children {
|
||||
if !branch.isIntrinsicallyAtomic() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// isIntrinsicallyAtomic reports whether a successful match of this node has no
|
||||
// alternative input-consuming path to explore if something later fails.
|
||||
func (n *RegexNode) isIntrinsicallyAtomic() bool {
|
||||
switch n.T {
|
||||
case NtOne, NtNotone, NtSet, NtMulti, NtRef, NtGrapheme,
|
||||
NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary,
|
||||
NtBeginning, NtStart, NtEndZ, NtEnd, NtNothing, NtEmpty, NtUpdateBumpalong,
|
||||
NtOneloopatomic, NtNotoneloopatomic, NtSetloopatomic,
|
||||
NtPosLook, NtNegLook, NtAtomic:
|
||||
return true
|
||||
|
||||
case NtOneloop, NtNotoneloop, NtSetloop, NtOnelazy, NtNotonelazy, NtSetlazy:
|
||||
return n.M == n.N
|
||||
|
||||
case NtCapture, NtGroup:
|
||||
return len(n.Children) == 1 && n.Children[0].isIntrinsicallyAtomic()
|
||||
|
||||
case NtConcatenate:
|
||||
for _, child := range n.Children {
|
||||
if !child.isIntrinsicallyAtomic() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (n *RegexNode) makeLoopAtomic() {
|
||||
|
||||
switch n.T {
|
||||
@@ -1680,6 +1789,12 @@ func (n *RegexNode) reduceConcatenationWithAdjacentStrings() {
|
||||
// Nested repeaters just get multiplied with each other if they're not
|
||||
// too lumpy
|
||||
func (n *RegexNode) reduceRep() *RegexNode {
|
||||
// Repeating a literal empty expression has no observable effect. In addition to
|
||||
// being unnecessary, retaining the loop would require one backtracking frame per
|
||||
// mandatory iteration even though no input is consumed.
|
||||
if len(n.Children) == 1 && n.Children[0].T == NtEmpty {
|
||||
return n.Children[0]
|
||||
}
|
||||
|
||||
u := n
|
||||
t := n.T
|
||||
@@ -1865,7 +1980,7 @@ func (n *RegexNode) makeQuantifier(lazy bool, min, max int) *RegexNode {
|
||||
// If the result is 0, there is no minimum we can enforce.
|
||||
func (n *RegexNode) ComputeMinLength() int {
|
||||
switch n.T {
|
||||
case NtOne, NtNotone, NtSet:
|
||||
case NtOne, NtNotone, NtSet, NtGrapheme:
|
||||
// single char
|
||||
return 1
|
||||
case NtMulti:
|
||||
@@ -1937,6 +2052,8 @@ func (n *RegexNode) computeMaxLength() int {
|
||||
switch n.T {
|
||||
case NtOne, NtNotone, NtSet:
|
||||
return 1
|
||||
case NtGrapheme:
|
||||
return -1
|
||||
case NtMulti:
|
||||
return len(n.Str)
|
||||
case NtNotonelazy, NtNotoneloop, NtNotoneloopatomic,
|
||||
@@ -2049,6 +2166,7 @@ var typeStr = []string{
|
||||
"ECMABoundary", "NonECMABoundary",
|
||||
"OneloopAtomic", "NotoneloopAtomic", "SetloopAtomic",
|
||||
"UpdateBumpalong",
|
||||
"Grapheme",
|
||||
}
|
||||
|
||||
func (n *RegexNode) Description() string {
|
||||
|
||||
+536
@@ -41,6 +41,9 @@ var unicodeAliasCategories = map[string]*unicode.RangeTable{
|
||||
"Grapheme_Cluster_Break=T": unicodeAliasGrapheme_Cluster_Break_T,
|
||||
"Grapheme_Cluster_Break=V": unicodeAliasGrapheme_Cluster_Break_V,
|
||||
"Grapheme_Cluster_Break=ZWJ": unicodeAliasGrapheme_Cluster_Break_ZWJ,
|
||||
"Indic_Conjunct_Break=Consonant": unicodeAliasIndic_Conjunct_Break_Consonant,
|
||||
"Indic_Conjunct_Break=Extend": unicodeAliasIndic_Conjunct_Break_Extend,
|
||||
"Indic_Conjunct_Break=Linker": unicodeAliasIndic_Conjunct_Break_Linker,
|
||||
"Math": unicodeAliasMath,
|
||||
"Sentence_Break=ATerm": unicodeAliasSentence_Break_ATerm,
|
||||
"Sentence_Break=CR": unicodeAliasSentence_Break_CR,
|
||||
@@ -90,6 +93,8 @@ var unicodeSupportedPropertyAliases = map[string]string{
|
||||
"extpict": "Extended_Pictographic",
|
||||
"gcb": "Grapheme_Cluster_Break",
|
||||
"graphemeclusterbreak": "Grapheme_Cluster_Break",
|
||||
"incb": "Indic_Conjunct_Break",
|
||||
"indicconjunctbreak": "Indic_Conjunct_Break",
|
||||
"math": "Math",
|
||||
"sb": "Sentence_Break",
|
||||
"sentencebreak": "Sentence_Break",
|
||||
@@ -118,6 +123,11 @@ var unicodeSupportedPropertyValueAliases = map[string]map[string]string{
|
||||
"v": "V",
|
||||
"zwj": "ZWJ",
|
||||
},
|
||||
"Indic_Conjunct_Break": {
|
||||
"consonant": "Consonant",
|
||||
"extend": "Extend",
|
||||
"linker": "Linker",
|
||||
},
|
||||
"Sentence_Break": {
|
||||
"at": "ATerm",
|
||||
"aterm": "ATerm",
|
||||
@@ -6056,3 +6066,529 @@ var unicodeAliasWord_Break_ZWJ = &unicode.RangeTable{
|
||||
{Lo: 0x200D, Hi: 0x200D, Stride: 1},
|
||||
},
|
||||
}
|
||||
|
||||
var unicodeAliasIndic_Conjunct_Break_Consonant = &unicode.RangeTable{
|
||||
R16: []unicode.Range16{
|
||||
{Lo: 0x915, Hi: 0x939, Stride: 1},
|
||||
{Lo: 0x958, Hi: 0x95F, Stride: 1},
|
||||
{Lo: 0x978, Hi: 0x97F, Stride: 1},
|
||||
{Lo: 0x995, Hi: 0x9A8, Stride: 1},
|
||||
{Lo: 0x9AA, Hi: 0x9B0, Stride: 1},
|
||||
{Lo: 0x9B2, Hi: 0x9B2, Stride: 1},
|
||||
{Lo: 0x9B6, Hi: 0x9B9, Stride: 1},
|
||||
{Lo: 0x9DC, Hi: 0x9DD, Stride: 1},
|
||||
{Lo: 0x9DF, Hi: 0x9DF, Stride: 1},
|
||||
{Lo: 0x9F0, Hi: 0x9F1, Stride: 1},
|
||||
{Lo: 0xA95, Hi: 0xAA8, Stride: 1},
|
||||
{Lo: 0xAAA, Hi: 0xAB0, Stride: 1},
|
||||
{Lo: 0xAB2, Hi: 0xAB3, Stride: 1},
|
||||
{Lo: 0xAB5, Hi: 0xAB9, Stride: 1},
|
||||
{Lo: 0xAF9, Hi: 0xAF9, Stride: 1},
|
||||
{Lo: 0xB15, Hi: 0xB28, Stride: 1},
|
||||
{Lo: 0xB2A, Hi: 0xB30, Stride: 1},
|
||||
{Lo: 0xB32, Hi: 0xB33, Stride: 1},
|
||||
{Lo: 0xB35, Hi: 0xB39, Stride: 1},
|
||||
{Lo: 0xB5C, Hi: 0xB5D, Stride: 1},
|
||||
{Lo: 0xB5F, Hi: 0xB5F, Stride: 1},
|
||||
{Lo: 0xB71, Hi: 0xB71, Stride: 1},
|
||||
{Lo: 0xC15, Hi: 0xC28, Stride: 1},
|
||||
{Lo: 0xC2A, Hi: 0xC39, Stride: 1},
|
||||
{Lo: 0xC58, Hi: 0xC5A, Stride: 1},
|
||||
{Lo: 0xD15, Hi: 0xD3A, Stride: 1},
|
||||
{Lo: 0x1000, Hi: 0x102A, Stride: 1},
|
||||
{Lo: 0x103F, Hi: 0x103F, Stride: 1},
|
||||
{Lo: 0x1050, Hi: 0x1055, Stride: 1},
|
||||
{Lo: 0x105A, Hi: 0x105D, Stride: 1},
|
||||
{Lo: 0x1061, Hi: 0x1061, Stride: 1},
|
||||
{Lo: 0x1065, Hi: 0x1066, Stride: 1},
|
||||
{Lo: 0x106E, Hi: 0x1070, Stride: 1},
|
||||
{Lo: 0x1075, Hi: 0x1081, Stride: 1},
|
||||
{Lo: 0x108E, Hi: 0x108E, Stride: 1},
|
||||
{Lo: 0x1780, Hi: 0x17B3, Stride: 1},
|
||||
{Lo: 0x1A20, Hi: 0x1A54, Stride: 1},
|
||||
{Lo: 0x1B0B, Hi: 0x1B0C, Stride: 1},
|
||||
{Lo: 0x1B13, Hi: 0x1B33, Stride: 1},
|
||||
{Lo: 0x1B45, Hi: 0x1B4C, Stride: 1},
|
||||
{Lo: 0x1B83, Hi: 0x1BA0, Stride: 1},
|
||||
{Lo: 0x1BAE, Hi: 0x1BAF, Stride: 1},
|
||||
{Lo: 0x1BBB, Hi: 0x1BBD, Stride: 1},
|
||||
{Lo: 0xA989, Hi: 0xA98B, Stride: 1},
|
||||
{Lo: 0xA98F, Hi: 0xA9B2, Stride: 1},
|
||||
{Lo: 0xA9E0, Hi: 0xA9E4, Stride: 1},
|
||||
{Lo: 0xA9E7, Hi: 0xA9EF, Stride: 1},
|
||||
{Lo: 0xA9FA, Hi: 0xA9FE, Stride: 1},
|
||||
{Lo: 0xAA60, Hi: 0xAA6F, Stride: 1},
|
||||
{Lo: 0xAA71, Hi: 0xAA73, Stride: 1},
|
||||
{Lo: 0xAA7A, Hi: 0xAA7A, Stride: 1},
|
||||
{Lo: 0xAA7E, Hi: 0xAA7F, Stride: 1},
|
||||
{Lo: 0xAAE0, Hi: 0xAAEA, Stride: 1},
|
||||
{Lo: 0xABC0, Hi: 0xABDA, Stride: 1},
|
||||
},
|
||||
R32: []unicode.Range32{
|
||||
{Lo: 0x10A00, Hi: 0x10A00, Stride: 1},
|
||||
{Lo: 0x10A10, Hi: 0x10A13, Stride: 1},
|
||||
{Lo: 0x10A15, Hi: 0x10A17, Stride: 1},
|
||||
{Lo: 0x10A19, Hi: 0x10A35, Stride: 1},
|
||||
{Lo: 0x11103, Hi: 0x11126, Stride: 1},
|
||||
{Lo: 0x11144, Hi: 0x11144, Stride: 1},
|
||||
{Lo: 0x11147, Hi: 0x11147, Stride: 1},
|
||||
{Lo: 0x11380, Hi: 0x11389, Stride: 1},
|
||||
{Lo: 0x1138B, Hi: 0x1138B, Stride: 1},
|
||||
{Lo: 0x1138E, Hi: 0x1138E, Stride: 1},
|
||||
{Lo: 0x11390, Hi: 0x113B5, Stride: 1},
|
||||
{Lo: 0x11900, Hi: 0x11906, Stride: 1},
|
||||
{Lo: 0x11909, Hi: 0x11909, Stride: 1},
|
||||
{Lo: 0x1190C, Hi: 0x11913, Stride: 1},
|
||||
{Lo: 0x11915, Hi: 0x11916, Stride: 1},
|
||||
{Lo: 0x11918, Hi: 0x1192F, Stride: 1},
|
||||
{Lo: 0x11A00, Hi: 0x11A00, Stride: 1},
|
||||
{Lo: 0x11A0B, Hi: 0x11A32, Stride: 1},
|
||||
{Lo: 0x11A50, Hi: 0x11A50, Stride: 1},
|
||||
{Lo: 0x11A5C, Hi: 0x11A83, Stride: 1},
|
||||
{Lo: 0x11F04, Hi: 0x11F10, Stride: 1},
|
||||
{Lo: 0x11F12, Hi: 0x11F33, Stride: 1},
|
||||
},
|
||||
}
|
||||
|
||||
var unicodeAliasIndic_Conjunct_Break_Extend = &unicode.RangeTable{
|
||||
R16: []unicode.Range16{
|
||||
{Lo: 0x300, Hi: 0x36F, Stride: 1},
|
||||
{Lo: 0x483, Hi: 0x487, Stride: 1},
|
||||
{Lo: 0x488, Hi: 0x489, Stride: 1},
|
||||
{Lo: 0x591, Hi: 0x5BD, Stride: 1},
|
||||
{Lo: 0x5BF, Hi: 0x5BF, Stride: 1},
|
||||
{Lo: 0x5C1, Hi: 0x5C2, Stride: 1},
|
||||
{Lo: 0x5C4, Hi: 0x5C5, Stride: 1},
|
||||
{Lo: 0x5C7, Hi: 0x5C7, Stride: 1},
|
||||
{Lo: 0x610, Hi: 0x61A, Stride: 1},
|
||||
{Lo: 0x64B, Hi: 0x65F, Stride: 1},
|
||||
{Lo: 0x670, Hi: 0x670, Stride: 1},
|
||||
{Lo: 0x6D6, Hi: 0x6DC, Stride: 1},
|
||||
{Lo: 0x6DF, Hi: 0x6E4, Stride: 1},
|
||||
{Lo: 0x6E7, Hi: 0x6E8, Stride: 1},
|
||||
{Lo: 0x6EA, Hi: 0x6ED, Stride: 1},
|
||||
{Lo: 0x711, Hi: 0x711, Stride: 1},
|
||||
{Lo: 0x730, Hi: 0x74A, Stride: 1},
|
||||
{Lo: 0x7A6, Hi: 0x7B0, Stride: 1},
|
||||
{Lo: 0x7EB, Hi: 0x7F3, Stride: 1},
|
||||
{Lo: 0x7FD, Hi: 0x7FD, Stride: 1},
|
||||
{Lo: 0x816, Hi: 0x819, Stride: 1},
|
||||
{Lo: 0x81B, Hi: 0x823, Stride: 1},
|
||||
{Lo: 0x825, Hi: 0x827, Stride: 1},
|
||||
{Lo: 0x829, Hi: 0x82D, Stride: 1},
|
||||
{Lo: 0x859, Hi: 0x85B, Stride: 1},
|
||||
{Lo: 0x897, Hi: 0x89F, Stride: 1},
|
||||
{Lo: 0x8CA, Hi: 0x8E1, Stride: 1},
|
||||
{Lo: 0x8E3, Hi: 0x902, Stride: 1},
|
||||
{Lo: 0x93A, Hi: 0x93A, Stride: 1},
|
||||
{Lo: 0x93C, Hi: 0x93C, Stride: 1},
|
||||
{Lo: 0x941, Hi: 0x948, Stride: 1},
|
||||
{Lo: 0x951, Hi: 0x957, Stride: 1},
|
||||
{Lo: 0x962, Hi: 0x963, Stride: 1},
|
||||
{Lo: 0x981, Hi: 0x981, Stride: 1},
|
||||
{Lo: 0x9BC, Hi: 0x9BC, Stride: 1},
|
||||
{Lo: 0x9BE, Hi: 0x9BE, Stride: 1},
|
||||
{Lo: 0x9C1, Hi: 0x9C4, Stride: 1},
|
||||
{Lo: 0x9D7, Hi: 0x9D7, Stride: 1},
|
||||
{Lo: 0x9E2, Hi: 0x9E3, Stride: 1},
|
||||
{Lo: 0x9FE, Hi: 0x9FE, Stride: 1},
|
||||
{Lo: 0xA01, Hi: 0xA02, Stride: 1},
|
||||
{Lo: 0xA3C, Hi: 0xA3C, Stride: 1},
|
||||
{Lo: 0xA41, Hi: 0xA42, Stride: 1},
|
||||
{Lo: 0xA47, Hi: 0xA48, Stride: 1},
|
||||
{Lo: 0xA4B, Hi: 0xA4D, Stride: 1},
|
||||
{Lo: 0xA51, Hi: 0xA51, Stride: 1},
|
||||
{Lo: 0xA70, Hi: 0xA71, Stride: 1},
|
||||
{Lo: 0xA75, Hi: 0xA75, Stride: 1},
|
||||
{Lo: 0xA81, Hi: 0xA82, Stride: 1},
|
||||
{Lo: 0xABC, Hi: 0xABC, Stride: 1},
|
||||
{Lo: 0xAC1, Hi: 0xAC5, Stride: 1},
|
||||
{Lo: 0xAC7, Hi: 0xAC8, Stride: 1},
|
||||
{Lo: 0xAE2, Hi: 0xAE3, Stride: 1},
|
||||
{Lo: 0xAFA, Hi: 0xAFF, Stride: 1},
|
||||
{Lo: 0xB01, Hi: 0xB01, Stride: 1},
|
||||
{Lo: 0xB3C, Hi: 0xB3C, Stride: 1},
|
||||
{Lo: 0xB3E, Hi: 0xB3E, Stride: 1},
|
||||
{Lo: 0xB3F, Hi: 0xB3F, Stride: 1},
|
||||
{Lo: 0xB41, Hi: 0xB44, Stride: 1},
|
||||
{Lo: 0xB55, Hi: 0xB56, Stride: 1},
|
||||
{Lo: 0xB57, Hi: 0xB57, Stride: 1},
|
||||
{Lo: 0xB62, Hi: 0xB63, Stride: 1},
|
||||
{Lo: 0xB82, Hi: 0xB82, Stride: 1},
|
||||
{Lo: 0xBBE, Hi: 0xBBE, Stride: 1},
|
||||
{Lo: 0xBC0, Hi: 0xBC0, Stride: 1},
|
||||
{Lo: 0xBCD, Hi: 0xBCD, Stride: 1},
|
||||
{Lo: 0xBD7, Hi: 0xBD7, Stride: 1},
|
||||
{Lo: 0xC00, Hi: 0xC00, Stride: 1},
|
||||
{Lo: 0xC04, Hi: 0xC04, Stride: 1},
|
||||
{Lo: 0xC3C, Hi: 0xC3C, Stride: 1},
|
||||
{Lo: 0xC3E, Hi: 0xC40, Stride: 1},
|
||||
{Lo: 0xC46, Hi: 0xC48, Stride: 1},
|
||||
{Lo: 0xC4A, Hi: 0xC4C, Stride: 1},
|
||||
{Lo: 0xC55, Hi: 0xC56, Stride: 1},
|
||||
{Lo: 0xC62, Hi: 0xC63, Stride: 1},
|
||||
{Lo: 0xC81, Hi: 0xC81, Stride: 1},
|
||||
{Lo: 0xCBC, Hi: 0xCBC, Stride: 1},
|
||||
{Lo: 0xCBF, Hi: 0xCBF, Stride: 1},
|
||||
{Lo: 0xCC0, Hi: 0xCC0, Stride: 1},
|
||||
{Lo: 0xCC2, Hi: 0xCC2, Stride: 1},
|
||||
{Lo: 0xCC6, Hi: 0xCC6, Stride: 1},
|
||||
{Lo: 0xCC7, Hi: 0xCC8, Stride: 1},
|
||||
{Lo: 0xCCA, Hi: 0xCCB, Stride: 1},
|
||||
{Lo: 0xCCC, Hi: 0xCCD, Stride: 1},
|
||||
{Lo: 0xCD5, Hi: 0xCD6, Stride: 1},
|
||||
{Lo: 0xCE2, Hi: 0xCE3, Stride: 1},
|
||||
{Lo: 0xD00, Hi: 0xD01, Stride: 1},
|
||||
{Lo: 0xD3B, Hi: 0xD3C, Stride: 1},
|
||||
{Lo: 0xD3E, Hi: 0xD3E, Stride: 1},
|
||||
{Lo: 0xD41, Hi: 0xD44, Stride: 1},
|
||||
{Lo: 0xD57, Hi: 0xD57, Stride: 1},
|
||||
{Lo: 0xD62, Hi: 0xD63, Stride: 1},
|
||||
{Lo: 0xD81, Hi: 0xD81, Stride: 1},
|
||||
{Lo: 0xDCA, Hi: 0xDCA, Stride: 1},
|
||||
{Lo: 0xDCF, Hi: 0xDCF, Stride: 1},
|
||||
{Lo: 0xDD2, Hi: 0xDD4, Stride: 1},
|
||||
{Lo: 0xDD6, Hi: 0xDD6, Stride: 1},
|
||||
{Lo: 0xDDF, Hi: 0xDDF, Stride: 1},
|
||||
{Lo: 0xE31, Hi: 0xE31, Stride: 1},
|
||||
{Lo: 0xE34, Hi: 0xE3A, Stride: 1},
|
||||
{Lo: 0xE47, Hi: 0xE4E, Stride: 1},
|
||||
{Lo: 0xEB1, Hi: 0xEB1, Stride: 1},
|
||||
{Lo: 0xEB4, Hi: 0xEBC, Stride: 1},
|
||||
{Lo: 0xEC8, Hi: 0xECE, Stride: 1},
|
||||
{Lo: 0xF18, Hi: 0xF19, Stride: 1},
|
||||
{Lo: 0xF35, Hi: 0xF35, Stride: 1},
|
||||
{Lo: 0xF37, Hi: 0xF37, Stride: 1},
|
||||
{Lo: 0xF39, Hi: 0xF39, Stride: 1},
|
||||
{Lo: 0xF71, Hi: 0xF7E, Stride: 1},
|
||||
{Lo: 0xF80, Hi: 0xF84, Stride: 1},
|
||||
{Lo: 0xF86, Hi: 0xF87, Stride: 1},
|
||||
{Lo: 0xF8D, Hi: 0xF97, Stride: 1},
|
||||
{Lo: 0xF99, Hi: 0xFBC, Stride: 1},
|
||||
{Lo: 0xFC6, Hi: 0xFC6, Stride: 1},
|
||||
{Lo: 0x102D, Hi: 0x1030, Stride: 1},
|
||||
{Lo: 0x1032, Hi: 0x1037, Stride: 1},
|
||||
{Lo: 0x103A, Hi: 0x103A, Stride: 1},
|
||||
{Lo: 0x103D, Hi: 0x103E, Stride: 1},
|
||||
{Lo: 0x1058, Hi: 0x1059, Stride: 1},
|
||||
{Lo: 0x105E, Hi: 0x1060, Stride: 1},
|
||||
{Lo: 0x1071, Hi: 0x1074, Stride: 1},
|
||||
{Lo: 0x1082, Hi: 0x1082, Stride: 1},
|
||||
{Lo: 0x1085, Hi: 0x1086, Stride: 1},
|
||||
{Lo: 0x108D, Hi: 0x108D, Stride: 1},
|
||||
{Lo: 0x109D, Hi: 0x109D, Stride: 1},
|
||||
{Lo: 0x135D, Hi: 0x135F, Stride: 1},
|
||||
{Lo: 0x1712, Hi: 0x1714, Stride: 1},
|
||||
{Lo: 0x1715, Hi: 0x1715, Stride: 1},
|
||||
{Lo: 0x1732, Hi: 0x1733, Stride: 1},
|
||||
{Lo: 0x1734, Hi: 0x1734, Stride: 1},
|
||||
{Lo: 0x1752, Hi: 0x1753, Stride: 1},
|
||||
{Lo: 0x1772, Hi: 0x1773, Stride: 1},
|
||||
{Lo: 0x17B4, Hi: 0x17B5, Stride: 1},
|
||||
{Lo: 0x17B7, Hi: 0x17BD, Stride: 1},
|
||||
{Lo: 0x17C6, Hi: 0x17C6, Stride: 1},
|
||||
{Lo: 0x17C9, Hi: 0x17D1, Stride: 1},
|
||||
{Lo: 0x17D3, Hi: 0x17D3, Stride: 1},
|
||||
{Lo: 0x17DD, Hi: 0x17DD, Stride: 1},
|
||||
{Lo: 0x180B, Hi: 0x180D, Stride: 1},
|
||||
{Lo: 0x180F, Hi: 0x180F, Stride: 1},
|
||||
{Lo: 0x1885, Hi: 0x1886, Stride: 1},
|
||||
{Lo: 0x18A9, Hi: 0x18A9, Stride: 1},
|
||||
{Lo: 0x1920, Hi: 0x1922, Stride: 1},
|
||||
{Lo: 0x1927, Hi: 0x1928, Stride: 1},
|
||||
{Lo: 0x1932, Hi: 0x1932, Stride: 1},
|
||||
{Lo: 0x1939, Hi: 0x193B, Stride: 1},
|
||||
{Lo: 0x1A17, Hi: 0x1A18, Stride: 1},
|
||||
{Lo: 0x1A1B, Hi: 0x1A1B, Stride: 1},
|
||||
{Lo: 0x1A56, Hi: 0x1A56, Stride: 1},
|
||||
{Lo: 0x1A58, Hi: 0x1A5E, Stride: 1},
|
||||
{Lo: 0x1A62, Hi: 0x1A62, Stride: 1},
|
||||
{Lo: 0x1A65, Hi: 0x1A6C, Stride: 1},
|
||||
{Lo: 0x1A73, Hi: 0x1A7C, Stride: 1},
|
||||
{Lo: 0x1A7F, Hi: 0x1A7F, Stride: 1},
|
||||
{Lo: 0x1AB0, Hi: 0x1ABD, Stride: 1},
|
||||
{Lo: 0x1ABE, Hi: 0x1ABE, Stride: 1},
|
||||
{Lo: 0x1ABF, Hi: 0x1ADD, Stride: 1},
|
||||
{Lo: 0x1AE0, Hi: 0x1AEB, Stride: 1},
|
||||
{Lo: 0x1B00, Hi: 0x1B03, Stride: 1},
|
||||
{Lo: 0x1B34, Hi: 0x1B34, Stride: 1},
|
||||
{Lo: 0x1B35, Hi: 0x1B35, Stride: 1},
|
||||
{Lo: 0x1B36, Hi: 0x1B3A, Stride: 1},
|
||||
{Lo: 0x1B3B, Hi: 0x1B3B, Stride: 1},
|
||||
{Lo: 0x1B3C, Hi: 0x1B3C, Stride: 1},
|
||||
{Lo: 0x1B3D, Hi: 0x1B3D, Stride: 1},
|
||||
{Lo: 0x1B42, Hi: 0x1B42, Stride: 1},
|
||||
{Lo: 0x1B43, Hi: 0x1B43, Stride: 1},
|
||||
{Lo: 0x1B6B, Hi: 0x1B73, Stride: 1},
|
||||
{Lo: 0x1B80, Hi: 0x1B81, Stride: 1},
|
||||
{Lo: 0x1BA2, Hi: 0x1BA5, Stride: 1},
|
||||
{Lo: 0x1BA8, Hi: 0x1BA9, Stride: 1},
|
||||
{Lo: 0x1BAA, Hi: 0x1BAA, Stride: 1},
|
||||
{Lo: 0x1BAC, Hi: 0x1BAD, Stride: 1},
|
||||
{Lo: 0x1BE6, Hi: 0x1BE6, Stride: 1},
|
||||
{Lo: 0x1BE8, Hi: 0x1BE9, Stride: 1},
|
||||
{Lo: 0x1BED, Hi: 0x1BED, Stride: 1},
|
||||
{Lo: 0x1BEF, Hi: 0x1BF1, Stride: 1},
|
||||
{Lo: 0x1BF2, Hi: 0x1BF3, Stride: 1},
|
||||
{Lo: 0x1C2C, Hi: 0x1C33, Stride: 1},
|
||||
{Lo: 0x1C36, Hi: 0x1C37, Stride: 1},
|
||||
{Lo: 0x1CD0, Hi: 0x1CD2, Stride: 1},
|
||||
{Lo: 0x1CD4, Hi: 0x1CE0, Stride: 1},
|
||||
{Lo: 0x1CE2, Hi: 0x1CE8, Stride: 1},
|
||||
{Lo: 0x1CED, Hi: 0x1CED, Stride: 1},
|
||||
{Lo: 0x1CF4, Hi: 0x1CF4, Stride: 1},
|
||||
{Lo: 0x1CF8, Hi: 0x1CF9, Stride: 1},
|
||||
{Lo: 0x1DC0, Hi: 0x1DFF, Stride: 1},
|
||||
{Lo: 0x200D, Hi: 0x200D, Stride: 1},
|
||||
{Lo: 0x20D0, Hi: 0x20DC, Stride: 1},
|
||||
{Lo: 0x20DD, Hi: 0x20E0, Stride: 1},
|
||||
{Lo: 0x20E1, Hi: 0x20E1, Stride: 1},
|
||||
{Lo: 0x20E2, Hi: 0x20E4, Stride: 1},
|
||||
{Lo: 0x20E5, Hi: 0x20F0, Stride: 1},
|
||||
{Lo: 0x2CEF, Hi: 0x2CF1, Stride: 1},
|
||||
{Lo: 0x2D7F, Hi: 0x2D7F, Stride: 1},
|
||||
{Lo: 0x2DE0, Hi: 0x2DFF, Stride: 1},
|
||||
{Lo: 0x302A, Hi: 0x302D, Stride: 1},
|
||||
{Lo: 0x302E, Hi: 0x302F, Stride: 1},
|
||||
{Lo: 0x3099, Hi: 0x309A, Stride: 1},
|
||||
{Lo: 0xA66F, Hi: 0xA66F, Stride: 1},
|
||||
{Lo: 0xA670, Hi: 0xA672, Stride: 1},
|
||||
{Lo: 0xA674, Hi: 0xA67D, Stride: 1},
|
||||
{Lo: 0xA69E, Hi: 0xA69F, Stride: 1},
|
||||
{Lo: 0xA6F0, Hi: 0xA6F1, Stride: 1},
|
||||
{Lo: 0xA802, Hi: 0xA802, Stride: 1},
|
||||
{Lo: 0xA806, Hi: 0xA806, Stride: 1},
|
||||
{Lo: 0xA80B, Hi: 0xA80B, Stride: 1},
|
||||
{Lo: 0xA825, Hi: 0xA826, Stride: 1},
|
||||
{Lo: 0xA82C, Hi: 0xA82C, Stride: 1},
|
||||
{Lo: 0xA8C4, Hi: 0xA8C5, Stride: 1},
|
||||
{Lo: 0xA8E0, Hi: 0xA8F1, Stride: 1},
|
||||
{Lo: 0xA8FF, Hi: 0xA8FF, Stride: 1},
|
||||
{Lo: 0xA926, Hi: 0xA92D, Stride: 1},
|
||||
{Lo: 0xA947, Hi: 0xA951, Stride: 1},
|
||||
{Lo: 0xA953, Hi: 0xA953, Stride: 1},
|
||||
{Lo: 0xA980, Hi: 0xA982, Stride: 1},
|
||||
{Lo: 0xA9B3, Hi: 0xA9B3, Stride: 1},
|
||||
{Lo: 0xA9B6, Hi: 0xA9B9, Stride: 1},
|
||||
{Lo: 0xA9BC, Hi: 0xA9BD, Stride: 1},
|
||||
{Lo: 0xA9E5, Hi: 0xA9E5, Stride: 1},
|
||||
{Lo: 0xAA29, Hi: 0xAA2E, Stride: 1},
|
||||
{Lo: 0xAA31, Hi: 0xAA32, Stride: 1},
|
||||
{Lo: 0xAA35, Hi: 0xAA36, Stride: 1},
|
||||
{Lo: 0xAA43, Hi: 0xAA43, Stride: 1},
|
||||
{Lo: 0xAA4C, Hi: 0xAA4C, Stride: 1},
|
||||
{Lo: 0xAA7C, Hi: 0xAA7C, Stride: 1},
|
||||
{Lo: 0xAAB0, Hi: 0xAAB0, Stride: 1},
|
||||
{Lo: 0xAAB2, Hi: 0xAAB4, Stride: 1},
|
||||
{Lo: 0xAAB7, Hi: 0xAAB8, Stride: 1},
|
||||
{Lo: 0xAABE, Hi: 0xAABF, Stride: 1},
|
||||
{Lo: 0xAAC1, Hi: 0xAAC1, Stride: 1},
|
||||
{Lo: 0xAAEC, Hi: 0xAAED, Stride: 1},
|
||||
{Lo: 0xABE5, Hi: 0xABE5, Stride: 1},
|
||||
{Lo: 0xABE8, Hi: 0xABE8, Stride: 1},
|
||||
{Lo: 0xABED, Hi: 0xABED, Stride: 1},
|
||||
{Lo: 0xFB1E, Hi: 0xFB1E, Stride: 1},
|
||||
{Lo: 0xFE00, Hi: 0xFE0F, Stride: 1},
|
||||
{Lo: 0xFE20, Hi: 0xFE2F, Stride: 1},
|
||||
{Lo: 0xFF9E, Hi: 0xFF9F, Stride: 1},
|
||||
},
|
||||
R32: []unicode.Range32{
|
||||
{Lo: 0x101FD, Hi: 0x101FD, Stride: 1},
|
||||
{Lo: 0x102E0, Hi: 0x102E0, Stride: 1},
|
||||
{Lo: 0x10376, Hi: 0x1037A, Stride: 1},
|
||||
{Lo: 0x10A01, Hi: 0x10A03, Stride: 1},
|
||||
{Lo: 0x10A05, Hi: 0x10A06, Stride: 1},
|
||||
{Lo: 0x10A0C, Hi: 0x10A0F, Stride: 1},
|
||||
{Lo: 0x10A38, Hi: 0x10A3A, Stride: 1},
|
||||
{Lo: 0x10AE5, Hi: 0x10AE6, Stride: 1},
|
||||
{Lo: 0x10D24, Hi: 0x10D27, Stride: 1},
|
||||
{Lo: 0x10D69, Hi: 0x10D6D, Stride: 1},
|
||||
{Lo: 0x10EAB, Hi: 0x10EAC, Stride: 1},
|
||||
{Lo: 0x10EFA, Hi: 0x10EFF, Stride: 1},
|
||||
{Lo: 0x10F46, Hi: 0x10F50, Stride: 1},
|
||||
{Lo: 0x10F82, Hi: 0x10F85, Stride: 1},
|
||||
{Lo: 0x11001, Hi: 0x11001, Stride: 1},
|
||||
{Lo: 0x11038, Hi: 0x11046, Stride: 1},
|
||||
{Lo: 0x11070, Hi: 0x11070, Stride: 1},
|
||||
{Lo: 0x11073, Hi: 0x11074, Stride: 1},
|
||||
{Lo: 0x1107F, Hi: 0x11081, Stride: 1},
|
||||
{Lo: 0x110B3, Hi: 0x110B6, Stride: 1},
|
||||
{Lo: 0x110B9, Hi: 0x110BA, Stride: 1},
|
||||
{Lo: 0x110C2, Hi: 0x110C2, Stride: 1},
|
||||
{Lo: 0x11100, Hi: 0x11102, Stride: 1},
|
||||
{Lo: 0x11127, Hi: 0x1112B, Stride: 1},
|
||||
{Lo: 0x1112D, Hi: 0x11132, Stride: 1},
|
||||
{Lo: 0x11134, Hi: 0x11134, Stride: 1},
|
||||
{Lo: 0x11173, Hi: 0x11173, Stride: 1},
|
||||
{Lo: 0x11180, Hi: 0x11181, Stride: 1},
|
||||
{Lo: 0x111B6, Hi: 0x111BE, Stride: 1},
|
||||
{Lo: 0x111C0, Hi: 0x111C0, Stride: 1},
|
||||
{Lo: 0x111C9, Hi: 0x111CC, Stride: 1},
|
||||
{Lo: 0x111CF, Hi: 0x111CF, Stride: 1},
|
||||
{Lo: 0x1122F, Hi: 0x11231, Stride: 1},
|
||||
{Lo: 0x11234, Hi: 0x11234, Stride: 1},
|
||||
{Lo: 0x11235, Hi: 0x11235, Stride: 1},
|
||||
{Lo: 0x11236, Hi: 0x11237, Stride: 1},
|
||||
{Lo: 0x1123E, Hi: 0x1123E, Stride: 1},
|
||||
{Lo: 0x11241, Hi: 0x11241, Stride: 1},
|
||||
{Lo: 0x112DF, Hi: 0x112DF, Stride: 1},
|
||||
{Lo: 0x112E3, Hi: 0x112EA, Stride: 1},
|
||||
{Lo: 0x11300, Hi: 0x11301, Stride: 1},
|
||||
{Lo: 0x1133B, Hi: 0x1133C, Stride: 1},
|
||||
{Lo: 0x1133E, Hi: 0x1133E, Stride: 1},
|
||||
{Lo: 0x11340, Hi: 0x11340, Stride: 1},
|
||||
{Lo: 0x1134D, Hi: 0x1134D, Stride: 1},
|
||||
{Lo: 0x11357, Hi: 0x11357, Stride: 1},
|
||||
{Lo: 0x11366, Hi: 0x1136C, Stride: 1},
|
||||
{Lo: 0x11370, Hi: 0x11374, Stride: 1},
|
||||
{Lo: 0x113B8, Hi: 0x113B8, Stride: 1},
|
||||
{Lo: 0x113BB, Hi: 0x113C0, Stride: 1},
|
||||
{Lo: 0x113C2, Hi: 0x113C2, Stride: 1},
|
||||
{Lo: 0x113C5, Hi: 0x113C5, Stride: 1},
|
||||
{Lo: 0x113C7, Hi: 0x113C9, Stride: 1},
|
||||
{Lo: 0x113CE, Hi: 0x113CE, Stride: 1},
|
||||
{Lo: 0x113CF, Hi: 0x113CF, Stride: 1},
|
||||
{Lo: 0x113D2, Hi: 0x113D2, Stride: 1},
|
||||
{Lo: 0x113E1, Hi: 0x113E2, Stride: 1},
|
||||
{Lo: 0x11438, Hi: 0x1143F, Stride: 1},
|
||||
{Lo: 0x11442, Hi: 0x11444, Stride: 1},
|
||||
{Lo: 0x11446, Hi: 0x11446, Stride: 1},
|
||||
{Lo: 0x1145E, Hi: 0x1145E, Stride: 1},
|
||||
{Lo: 0x114B0, Hi: 0x114B0, Stride: 1},
|
||||
{Lo: 0x114B3, Hi: 0x114B8, Stride: 1},
|
||||
{Lo: 0x114BA, Hi: 0x114BA, Stride: 1},
|
||||
{Lo: 0x114BD, Hi: 0x114BD, Stride: 1},
|
||||
{Lo: 0x114BF, Hi: 0x114C0, Stride: 1},
|
||||
{Lo: 0x114C2, Hi: 0x114C3, Stride: 1},
|
||||
{Lo: 0x115AF, Hi: 0x115AF, Stride: 1},
|
||||
{Lo: 0x115B2, Hi: 0x115B5, Stride: 1},
|
||||
{Lo: 0x115BC, Hi: 0x115BD, Stride: 1},
|
||||
{Lo: 0x115BF, Hi: 0x115C0, Stride: 1},
|
||||
{Lo: 0x115DC, Hi: 0x115DD, Stride: 1},
|
||||
{Lo: 0x11633, Hi: 0x1163A, Stride: 1},
|
||||
{Lo: 0x1163D, Hi: 0x1163D, Stride: 1},
|
||||
{Lo: 0x1163F, Hi: 0x11640, Stride: 1},
|
||||
{Lo: 0x116AB, Hi: 0x116AB, Stride: 1},
|
||||
{Lo: 0x116AD, Hi: 0x116AD, Stride: 1},
|
||||
{Lo: 0x116B0, Hi: 0x116B5, Stride: 1},
|
||||
{Lo: 0x116B6, Hi: 0x116B6, Stride: 1},
|
||||
{Lo: 0x116B7, Hi: 0x116B7, Stride: 1},
|
||||
{Lo: 0x1171D, Hi: 0x1171D, Stride: 1},
|
||||
{Lo: 0x1171F, Hi: 0x1171F, Stride: 1},
|
||||
{Lo: 0x11722, Hi: 0x11725, Stride: 1},
|
||||
{Lo: 0x11727, Hi: 0x1172B, Stride: 1},
|
||||
{Lo: 0x1182F, Hi: 0x11837, Stride: 1},
|
||||
{Lo: 0x11839, Hi: 0x1183A, Stride: 1},
|
||||
{Lo: 0x11930, Hi: 0x11930, Stride: 1},
|
||||
{Lo: 0x1193B, Hi: 0x1193C, Stride: 1},
|
||||
{Lo: 0x1193D, Hi: 0x1193D, Stride: 1},
|
||||
{Lo: 0x11943, Hi: 0x11943, Stride: 1},
|
||||
{Lo: 0x119D4, Hi: 0x119D7, Stride: 1},
|
||||
{Lo: 0x119DA, Hi: 0x119DB, Stride: 1},
|
||||
{Lo: 0x119E0, Hi: 0x119E0, Stride: 1},
|
||||
{Lo: 0x11A01, Hi: 0x11A0A, Stride: 1},
|
||||
{Lo: 0x11A33, Hi: 0x11A38, Stride: 1},
|
||||
{Lo: 0x11A3B, Hi: 0x11A3E, Stride: 1},
|
||||
{Lo: 0x11A51, Hi: 0x11A56, Stride: 1},
|
||||
{Lo: 0x11A59, Hi: 0x11A5B, Stride: 1},
|
||||
{Lo: 0x11A8A, Hi: 0x11A96, Stride: 1},
|
||||
{Lo: 0x11A98, Hi: 0x11A98, Stride: 1},
|
||||
{Lo: 0x11B60, Hi: 0x11B60, Stride: 1},
|
||||
{Lo: 0x11B62, Hi: 0x11B64, Stride: 1},
|
||||
{Lo: 0x11B66, Hi: 0x11B66, Stride: 1},
|
||||
{Lo: 0x11C30, Hi: 0x11C36, Stride: 1},
|
||||
{Lo: 0x11C38, Hi: 0x11C3D, Stride: 1},
|
||||
{Lo: 0x11C3F, Hi: 0x11C3F, Stride: 1},
|
||||
{Lo: 0x11C92, Hi: 0x11CA7, Stride: 1},
|
||||
{Lo: 0x11CAA, Hi: 0x11CB0, Stride: 1},
|
||||
{Lo: 0x11CB2, Hi: 0x11CB3, Stride: 1},
|
||||
{Lo: 0x11CB5, Hi: 0x11CB6, Stride: 1},
|
||||
{Lo: 0x11D31, Hi: 0x11D36, Stride: 1},
|
||||
{Lo: 0x11D3A, Hi: 0x11D3A, Stride: 1},
|
||||
{Lo: 0x11D3C, Hi: 0x11D3D, Stride: 1},
|
||||
{Lo: 0x11D3F, Hi: 0x11D45, Stride: 1},
|
||||
{Lo: 0x11D47, Hi: 0x11D47, Stride: 1},
|
||||
{Lo: 0x11D90, Hi: 0x11D91, Stride: 1},
|
||||
{Lo: 0x11D95, Hi: 0x11D95, Stride: 1},
|
||||
{Lo: 0x11D97, Hi: 0x11D97, Stride: 1},
|
||||
{Lo: 0x11EF3, Hi: 0x11EF4, Stride: 1},
|
||||
{Lo: 0x11F00, Hi: 0x11F01, Stride: 1},
|
||||
{Lo: 0x11F36, Hi: 0x11F3A, Stride: 1},
|
||||
{Lo: 0x11F40, Hi: 0x11F40, Stride: 1},
|
||||
{Lo: 0x11F41, Hi: 0x11F41, Stride: 1},
|
||||
{Lo: 0x11F5A, Hi: 0x11F5A, Stride: 1},
|
||||
{Lo: 0x13440, Hi: 0x13440, Stride: 1},
|
||||
{Lo: 0x13447, Hi: 0x13455, Stride: 1},
|
||||
{Lo: 0x1611E, Hi: 0x16129, Stride: 1},
|
||||
{Lo: 0x1612D, Hi: 0x1612F, Stride: 1},
|
||||
{Lo: 0x16AF0, Hi: 0x16AF4, Stride: 1},
|
||||
{Lo: 0x16B30, Hi: 0x16B36, Stride: 1},
|
||||
{Lo: 0x16F4F, Hi: 0x16F4F, Stride: 1},
|
||||
{Lo: 0x16F8F, Hi: 0x16F92, Stride: 1},
|
||||
{Lo: 0x16FE4, Hi: 0x16FE4, Stride: 1},
|
||||
{Lo: 0x16FF0, Hi: 0x16FF1, Stride: 1},
|
||||
{Lo: 0x1BC9D, Hi: 0x1BC9E, Stride: 1},
|
||||
{Lo: 0x1CF00, Hi: 0x1CF2D, Stride: 1},
|
||||
{Lo: 0x1CF30, Hi: 0x1CF46, Stride: 1},
|
||||
{Lo: 0x1D165, Hi: 0x1D166, Stride: 1},
|
||||
{Lo: 0x1D167, Hi: 0x1D169, Stride: 1},
|
||||
{Lo: 0x1D16D, Hi: 0x1D172, Stride: 1},
|
||||
{Lo: 0x1D17B, Hi: 0x1D182, Stride: 1},
|
||||
{Lo: 0x1D185, Hi: 0x1D18B, Stride: 1},
|
||||
{Lo: 0x1D1AA, Hi: 0x1D1AD, Stride: 1},
|
||||
{Lo: 0x1D242, Hi: 0x1D244, Stride: 1},
|
||||
{Lo: 0x1DA00, Hi: 0x1DA36, Stride: 1},
|
||||
{Lo: 0x1DA3B, Hi: 0x1DA6C, Stride: 1},
|
||||
{Lo: 0x1DA75, Hi: 0x1DA75, Stride: 1},
|
||||
{Lo: 0x1DA84, Hi: 0x1DA84, Stride: 1},
|
||||
{Lo: 0x1DA9B, Hi: 0x1DA9F, Stride: 1},
|
||||
{Lo: 0x1DAA1, Hi: 0x1DAAF, Stride: 1},
|
||||
{Lo: 0x1E000, Hi: 0x1E006, Stride: 1},
|
||||
{Lo: 0x1E008, Hi: 0x1E018, Stride: 1},
|
||||
{Lo: 0x1E01B, Hi: 0x1E021, Stride: 1},
|
||||
{Lo: 0x1E023, Hi: 0x1E024, Stride: 1},
|
||||
{Lo: 0x1E026, Hi: 0x1E02A, Stride: 1},
|
||||
{Lo: 0x1E08F, Hi: 0x1E08F, Stride: 1},
|
||||
{Lo: 0x1E130, Hi: 0x1E136, Stride: 1},
|
||||
{Lo: 0x1E2AE, Hi: 0x1E2AE, Stride: 1},
|
||||
{Lo: 0x1E2EC, Hi: 0x1E2EF, Stride: 1},
|
||||
{Lo: 0x1E4EC, Hi: 0x1E4EF, Stride: 1},
|
||||
{Lo: 0x1E5EE, Hi: 0x1E5EF, Stride: 1},
|
||||
{Lo: 0x1E6E3, Hi: 0x1E6E3, Stride: 1},
|
||||
{Lo: 0x1E6E6, Hi: 0x1E6E6, Stride: 1},
|
||||
{Lo: 0x1E6EE, Hi: 0x1E6EF, Stride: 1},
|
||||
{Lo: 0x1E6F5, Hi: 0x1E6F5, Stride: 1},
|
||||
{Lo: 0x1E8D0, Hi: 0x1E8D6, Stride: 1},
|
||||
{Lo: 0x1E944, Hi: 0x1E94A, Stride: 1},
|
||||
{Lo: 0x1F3FB, Hi: 0x1F3FF, Stride: 1},
|
||||
{Lo: 0xE0020, Hi: 0xE007F, Stride: 1},
|
||||
{Lo: 0xE0100, Hi: 0xE01EF, Stride: 1},
|
||||
},
|
||||
}
|
||||
|
||||
var unicodeAliasIndic_Conjunct_Break_Linker = &unicode.RangeTable{
|
||||
R16: []unicode.Range16{
|
||||
{Lo: 0x94D, Hi: 0x94D, Stride: 1},
|
||||
{Lo: 0x9CD, Hi: 0x9CD, Stride: 1},
|
||||
{Lo: 0xACD, Hi: 0xACD, Stride: 1},
|
||||
{Lo: 0xB4D, Hi: 0xB4D, Stride: 1},
|
||||
{Lo: 0xC4D, Hi: 0xC4D, Stride: 1},
|
||||
{Lo: 0xD4D, Hi: 0xD4D, Stride: 1},
|
||||
{Lo: 0x1039, Hi: 0x1039, Stride: 1},
|
||||
{Lo: 0x17D2, Hi: 0x17D2, Stride: 1},
|
||||
{Lo: 0x1A60, Hi: 0x1A60, Stride: 1},
|
||||
{Lo: 0x1B44, Hi: 0x1B44, Stride: 1},
|
||||
{Lo: 0x1BAB, Hi: 0x1BAB, Stride: 1},
|
||||
{Lo: 0xA9C0, Hi: 0xA9C0, Stride: 1},
|
||||
{Lo: 0xAAF6, Hi: 0xAAF6, Stride: 1},
|
||||
},
|
||||
R32: []unicode.Range32{
|
||||
{Lo: 0x10A3F, Hi: 0x10A3F, Stride: 1},
|
||||
{Lo: 0x11133, Hi: 0x11133, Stride: 1},
|
||||
{Lo: 0x113D0, Hi: 0x113D0, Stride: 1},
|
||||
{Lo: 0x1193E, Hi: 0x1193E, Stride: 1},
|
||||
{Lo: 0x11A47, Hi: 0x11A47, Stride: 1},
|
||||
{Lo: 0x11A99, Hi: 0x11A99, Stride: 1},
|
||||
{Lo: 0x11F42, Hi: 0x11F42, Stride: 1},
|
||||
},
|
||||
}
|
||||
|
||||
+231
-24
@@ -4,34 +4,93 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
)
|
||||
|
||||
func Write(tree *RegexTree) (*Code, error) {
|
||||
w := writer{
|
||||
intStack: make([]int, 0, 32),
|
||||
emitted: make([]int, 2),
|
||||
stringhash: make(map[string]int),
|
||||
sethash: make(map[string]int),
|
||||
w := newWriter(nil)
|
||||
code, err := w.codeFromTree(tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
code, err := w.codeFromTree(tree)
|
||||
if slices.Contains(code.CaptureSlotInUse, false) {
|
||||
quickWriter := newWriter(code.CaptureSlotInUse)
|
||||
quickCode, err := quickWriter.codeFromTree(tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !dispatchTablesEqual(code.Dispatches, quickCode.Dispatches) {
|
||||
return nil, fmt.Errorf("full and quick dispatch tables differ")
|
||||
}
|
||||
for i := range quickCode.Dispatches {
|
||||
quickCode.Dispatches[i].Sets = code.Dispatches[i].Sets
|
||||
quickCode.Dispatches[i].ASCII = code.Dispatches[i].ASCII
|
||||
quickCode.Dispatches[i].ASCIIMasks = code.Dispatches[i].ASCIIMasks
|
||||
}
|
||||
code.QuickCodes = quickCode.Codes
|
||||
code.QuickDispatches = quickCode.Dispatches
|
||||
}
|
||||
return code, nil
|
||||
}
|
||||
|
||||
return code, err
|
||||
func newWriter(quickCaptureSlots []bool) writer {
|
||||
return writer{
|
||||
intStack: make([]int, 0, 32),
|
||||
emitted: make([]int, 2),
|
||||
stringhash: make(map[string]int),
|
||||
sethash: make(map[string]int),
|
||||
preconsumed: make(map[*RegexNode]bool),
|
||||
dispatchInfo: make(map[*RegexNode]dispatchInfo),
|
||||
dispatchCodes: make(map[*RegexNode]int),
|
||||
dispatchGotos: make(map[*RegexNode][]int),
|
||||
quickCaptureSlots: quickCaptureSlots,
|
||||
}
|
||||
}
|
||||
|
||||
type writer struct {
|
||||
emitted []int
|
||||
|
||||
intStack []int
|
||||
curpos int
|
||||
stringhash map[string]int
|
||||
stringtable [][]rune
|
||||
sethash map[string]int
|
||||
settable []*CharSet
|
||||
counting bool
|
||||
count int
|
||||
trackcount int
|
||||
caps map[int]int
|
||||
intStack []int
|
||||
curpos int
|
||||
stringhash map[string]int
|
||||
stringtable [][]rune
|
||||
sethash map[string]int
|
||||
settable []*CharSet
|
||||
dispatchtable []DispatchTable
|
||||
preconsumed map[*RegexNode]bool
|
||||
dispatchInfo map[*RegexNode]dispatchInfo
|
||||
dispatchCodes map[*RegexNode]int
|
||||
dispatchGotos map[*RegexNode][]int
|
||||
counting bool
|
||||
count int
|
||||
trackcount int
|
||||
caps map[int]int
|
||||
quickCaptureSlots []bool
|
||||
}
|
||||
|
||||
type dispatchInfo struct {
|
||||
sets []*CharSet
|
||||
leaders []*RegexNode
|
||||
complete []bool
|
||||
ok bool
|
||||
}
|
||||
|
||||
func dispatchTablesEqual(left, right []DispatchTable) bool {
|
||||
if len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
for i := range left {
|
||||
if !slices.Equal(left[i].Sets, right[i].Sets) ||
|
||||
!slices.Equal(left[i].ASCIIMasks, right[i].ASCIIMasks) ||
|
||||
(left[i].ASCII == nil) != (right[i].ASCII == nil) {
|
||||
return false
|
||||
}
|
||||
if left[i].ASCII != nil && *left[i].ASCII != *right[i].ASCII {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -74,6 +133,8 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) {
|
||||
for {
|
||||
if !w.counting {
|
||||
w.emitted = make([]int, w.count)
|
||||
w.dispatchCodes = make(map[*RegexNode]int)
|
||||
w.dispatchGotos = make(map[*RegexNode][]int)
|
||||
}
|
||||
|
||||
curNode = tree.Root
|
||||
@@ -120,6 +181,9 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) {
|
||||
|
||||
w.counting = false
|
||||
}
|
||||
if w.quickCaptureSlots != nil {
|
||||
return &Code{Codes: w.emitted, Dispatches: w.dispatchtable, TrackCount: w.trackcount}, nil
|
||||
}
|
||||
|
||||
fcPrefix := getFirstCharsPrefix(tree)
|
||||
prefix := getPrefix(tree)
|
||||
@@ -141,17 +205,57 @@ func (w *writer) codeFromTree(tree *RegexTree) (*Code, error) {
|
||||
Codes: w.emitted,
|
||||
Strings: w.stringtable,
|
||||
Sets: w.settable,
|
||||
Dispatches: w.dispatchtable,
|
||||
TrackCount: w.trackcount,
|
||||
Caps: w.caps,
|
||||
Capsize: capsize,
|
||||
CaptureSlotInUse: captureSlotsInUse(w.emitted, capsize),
|
||||
FcPrefix: fcPrefix,
|
||||
BmPrefix: bmPrefix,
|
||||
Anchors: getAnchors(tree),
|
||||
RightToLeft: rtl,
|
||||
FindOptimizations: tree.FindOptimizations,
|
||||
LeftContextRunes: AnalyzeLeftContext(tree.Root),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AnalyzeLeftContext returns how many runes before a candidate start the matcher
|
||||
// may inspect. Slicing the input down to that candidate is legal only when no
|
||||
// opcode depends on the original search origin or unbounded left text.
|
||||
//
|
||||
// 0 never looks left of the start position
|
||||
// 1 one previous rune is enough for \b, or to keep ^/\A from seeing
|
||||
// the candidate as the original start (they test leftchars()==0)
|
||||
// -1 do not slice: lookbehind, or \G (NtStart), which keys off textstart
|
||||
func AnalyzeLeftContext(n *RegexNode) int {
|
||||
if n == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
need := 0
|
||||
switch n.T {
|
||||
case NtPosLook, NtNegLook:
|
||||
if n.Options&RightToLeft != 0 {
|
||||
return -1
|
||||
}
|
||||
case NtStart:
|
||||
return -1
|
||||
case NtBol, NtBeginning, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary:
|
||||
need = 1
|
||||
}
|
||||
|
||||
for _, child := range n.Children {
|
||||
childNeed := AnalyzeLeftContext(child)
|
||||
if childNeed < 0 {
|
||||
return -1
|
||||
}
|
||||
if childNeed > need {
|
||||
need = childNeed
|
||||
}
|
||||
}
|
||||
return need
|
||||
}
|
||||
|
||||
// The main RegexCode generator. It does a depth-first walk
|
||||
// through the tree and calls EmitFragment to emits code before
|
||||
// and after each child of an interior node, and at each leaf.
|
||||
@@ -171,13 +275,37 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
case NtConcatenate | BeforeChild, NtConcatenate | AfterChild, NtEmpty:
|
||||
|
||||
case NtAlternate | BeforeChild:
|
||||
if curIndex < len(node.Children)-1 {
|
||||
if sets, leaders, _, ok := w.getDispatchCandidates(node); ok {
|
||||
if curIndex == 0 {
|
||||
w.dispatchCodes[node] = w.dispatchCode(sets)
|
||||
w.emit1(Dispatch|bits, w.dispatchCodes[node])
|
||||
}
|
||||
w.preconsumed[leaders[curIndex]] = true
|
||||
w.patchDispatchBranch(node, curIndex, w.curPos())
|
||||
} else if curIndex < len(node.Children)-1 {
|
||||
w.pushInt(w.curPos())
|
||||
w.emit1(Lazybranch, 0)
|
||||
}
|
||||
|
||||
case NtAlternate | AfterChild:
|
||||
if curIndex < len(node.Children)-1 {
|
||||
if _, _, complete, ok := w.getDispatchCandidates(node); ok {
|
||||
if curIndex < len(node.Children)-1 && !complete[curIndex] {
|
||||
gotoPos := w.curPos()
|
||||
w.emit1(Goto, 0)
|
||||
w.dispatchGotos[node] = append(w.dispatchGotos[node], gotoPos)
|
||||
}
|
||||
if curIndex == len(node.Children)-1 {
|
||||
end := w.curPos()
|
||||
for _, gotoPos := range w.dispatchGotos[node] {
|
||||
w.patchJump(gotoPos, end)
|
||||
}
|
||||
for branch, isComplete := range complete {
|
||||
if isComplete {
|
||||
w.patchDispatchBranch(node, branch, end)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if curIndex < len(node.Children)-1 {
|
||||
lbPos := w.popInt()
|
||||
w.pushInt(w.curPos())
|
||||
w.emit1(Goto, 0)
|
||||
@@ -281,10 +409,14 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
case NtGroup | BeforeChild, NtGroup | AfterChild:
|
||||
|
||||
case NtCapture | BeforeChild:
|
||||
w.emit(Setmark)
|
||||
if w.emitCapture(node) {
|
||||
w.emit(Setmark)
|
||||
}
|
||||
|
||||
case NtCapture | AfterChild:
|
||||
w.emit2(Capturemark, w.mapCapnum(node.M), w.mapCapnum(node.N))
|
||||
if w.emitCapture(node) {
|
||||
w.emit2(Capturemark, w.mapCapnum(node.M), w.mapCapnum(node.N))
|
||||
}
|
||||
|
||||
case NtPosLook | BeforeChild:
|
||||
// NOTE: the following line causes lookahead/lookbehind to be
|
||||
@@ -317,7 +449,9 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
w.emit(Forejump)
|
||||
|
||||
case NtOne, NtNotone:
|
||||
w.emit1(InstOp(node.T|ntBits), int(node.Ch))
|
||||
if !w.preconsumed[node] {
|
||||
w.emit1(InstOp(node.T|ntBits), int(node.Ch))
|
||||
}
|
||||
|
||||
case NtNotoneloop, NtNotoneloopatomic, NtNotonelazy, NtOneloop, NtOneloopatomic, NtOnelazy:
|
||||
if node.M > 0 {
|
||||
@@ -348,10 +482,22 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
}
|
||||
|
||||
case NtMulti:
|
||||
w.emit1(InstOp(node.T|ntBits), w.stringCode(node.Str))
|
||||
str := node.Str
|
||||
if w.preconsumed[node] {
|
||||
if bits&Rtl != 0 {
|
||||
str = str[:len(str)-1]
|
||||
} else {
|
||||
str = str[1:]
|
||||
}
|
||||
}
|
||||
if len(str) > 0 {
|
||||
w.emit1(InstOp(node.T|ntBits), w.stringCode(str))
|
||||
}
|
||||
|
||||
case NtSet:
|
||||
w.emit1(InstOp(node.T|ntBits), w.setCode(node.Set))
|
||||
if !w.preconsumed[node] {
|
||||
w.emit1(InstOp(node.T|ntBits), w.setCode(node.Set))
|
||||
}
|
||||
|
||||
case NtRef:
|
||||
w.emit1(InstOp(node.T|ntBits), w.mapCapnum(node.M))
|
||||
@@ -359,6 +505,9 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
case NtNothing, NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary, NtBeginning, NtStart, NtEndZ, NtEnd, NtUpdateBumpalong:
|
||||
w.emit(InstOp(node.T))
|
||||
|
||||
case NtGrapheme:
|
||||
w.emit(InstOp(node.T | ntBits))
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unexpected opcode in regular expression generation: %v", nodetype)
|
||||
}
|
||||
@@ -366,6 +515,17 @@ func (w *writer) emitFragment(nodetype NodeType, node *RegexNode, curIndex int)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *writer) emitCapture(node *RegexNode) bool {
|
||||
if w.quickCaptureSlots == nil {
|
||||
return true
|
||||
}
|
||||
capnum, uncapnum := w.mapCapnum(node.M), w.mapCapnum(node.N)
|
||||
if uncapnum != -1 {
|
||||
return true
|
||||
}
|
||||
return capnum >= 0 && (capnum >= len(w.quickCaptureSlots) || w.quickCaptureSlots[capnum])
|
||||
}
|
||||
|
||||
// To avoid recursion, we use a simple integer stack.
|
||||
// This is the push.
|
||||
func (w *writer) pushInt(i int) {
|
||||
@@ -398,6 +558,53 @@ func (w *writer) patchJump(offset, jumpDest int) {
|
||||
w.emitted[offset+1] = jumpDest
|
||||
}
|
||||
|
||||
func (w *writer) getDispatchCandidates(node *RegexNode) ([]*CharSet, []*RegexNode, []bool, bool) {
|
||||
if info, found := w.dispatchInfo[node]; found {
|
||||
return info.sets, info.leaders, info.complete, info.ok
|
||||
}
|
||||
sets, leaders, complete, ok := node.dispatchCandidates()
|
||||
w.dispatchInfo[node] = dispatchInfo{sets: sets, leaders: leaders, complete: complete, ok: ok}
|
||||
return sets, leaders, complete, ok
|
||||
}
|
||||
|
||||
func (w *writer) dispatchCode(sets []*CharSet) int {
|
||||
if w.counting {
|
||||
return 0
|
||||
}
|
||||
table := DispatchTable{
|
||||
Sets: make([]int, len(sets)),
|
||||
Branches: make([]int, len(sets)),
|
||||
}
|
||||
if len(sets) >= 4 {
|
||||
table.ASCII = &[128]uint16{}
|
||||
} else {
|
||||
table.ASCIIMasks = make([]uint64, len(sets)*2)
|
||||
}
|
||||
for i, set := range sets {
|
||||
table.Sets[i] = w.setCode(set)
|
||||
for ch := rune(0); ch < 128; ch++ {
|
||||
if set.CharIn(ch) {
|
||||
if table.ASCII != nil {
|
||||
table.ASCII[ch] = uint16(i + 1)
|
||||
} else {
|
||||
table.ASCIIMasks[i*2+int(ch>>6)] |= uint64(1) << (ch & 63)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
index := len(w.dispatchtable)
|
||||
w.dispatchtable = append(w.dispatchtable, table)
|
||||
return index
|
||||
}
|
||||
|
||||
func (w *writer) patchDispatchBranch(node *RegexNode, branch, target int) {
|
||||
if w.counting {
|
||||
return
|
||||
}
|
||||
table := w.dispatchCodes[node]
|
||||
w.dispatchtable[table].Branches[branch] = target
|
||||
}
|
||||
|
||||
// Returns an index in the set table for a charset
|
||||
// uses a map to eliminate duplicates.
|
||||
func (w *writer) setCode(set *CharSet) int {
|
||||
|
||||
+20
-11
@@ -8,6 +8,9 @@
|
||||
// In case of default TTL (10 years) and default MaxSize (0, unlimited) the cache will be truly unlimited
|
||||
// and will never delete entries from itself automatically.
|
||||
//
|
||||
// Get, Peek, Contains and Values treat an expired entry as missing, while Len and Keys report
|
||||
// everything the cache still holds, expired entries included.
|
||||
//
|
||||
// Important: only reliable way of not having expired entries stuck in a cache is to
|
||||
// run cache.DeleteExpired periodically using time.Ticker, advisable period is 1/2 of TTL.
|
||||
package cache
|
||||
@@ -78,9 +81,10 @@ func NewCache[K comparable, V any]() Cache[K, V] {
|
||||
}
|
||||
}
|
||||
|
||||
// Add adds a value to the cache. Returns true if an eviction occurred.
|
||||
// Add adds a value to the cache. Returns true if an eviction occurred, either
|
||||
// because the size was exceeded or because the oldest entry expired.
|
||||
// Returns false if there was no eviction: the item was already in the cache,
|
||||
// or the size was not exceeded.
|
||||
// or nothing had to be removed.
|
||||
func (c *cacheImpl[K, V]) Add(key K, value V) (evicted bool) {
|
||||
return c.addWithTTL(key, value, c.ttl)
|
||||
}
|
||||
@@ -90,9 +94,10 @@ func (c *cacheImpl[K, V]) Set(key K, value V, ttl time.Duration) {
|
||||
c.addWithTTL(key, value, ttl)
|
||||
}
|
||||
|
||||
// Returns true if an eviction occurred.
|
||||
// Returns true if an eviction occurred, either because the size was exceeded
|
||||
// or because the oldest entry expired.
|
||||
// Returns false if there was no eviction: the item was already in the cache,
|
||||
// or the size was not exceeded.
|
||||
// or nothing had to be removed.
|
||||
func (c *cacheImpl[K, V]) addWithTTL(key K, value V, ttl time.Duration) (evicted bool) {
|
||||
if ttl == 0 {
|
||||
ttl = c.ttl
|
||||
@@ -119,15 +124,16 @@ func (c *cacheImpl[K, V]) addWithTTL(key K, value V, ttl time.Duration) (evicted
|
||||
ent := c.evictList.Back()
|
||||
if ent != nil && now.After(ent.Value.(*cacheItem[K, V]).expiresAt) {
|
||||
c.removeElement(ent)
|
||||
evicted = true
|
||||
}
|
||||
}
|
||||
|
||||
evict := c.maxKeys > 0 && len(c.items) > c.maxKeys
|
||||
// Verify size not exceeded
|
||||
if evict {
|
||||
if c.maxKeys > 0 && len(c.items) > c.maxKeys {
|
||||
c.removeOldest()
|
||||
evicted = true
|
||||
}
|
||||
return evict
|
||||
return evicted
|
||||
}
|
||||
|
||||
// Get returns the key value if it's not expired
|
||||
@@ -152,12 +158,15 @@ func (c *cacheImpl[K, V]) Get(key K) (V, bool) {
|
||||
}
|
||||
|
||||
// Contains checks if a key is in the cache, without updating the recent-ness
|
||||
// or deleting it for being stale.
|
||||
// or deleting it for being stale. Expired entries are reported as missing.
|
||||
func (c *cacheImpl[K, V]) Contains(key K) (ok bool) {
|
||||
c.Lock()
|
||||
defer c.Unlock()
|
||||
_, ok = c.items[key]
|
||||
return ok
|
||||
ent, ok := c.items[key]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return !time.Now().After(ent.Value.(*cacheItem[K, V]).expiresAt)
|
||||
}
|
||||
|
||||
// Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key.
|
||||
@@ -199,10 +208,10 @@ func (c *cacheImpl[K, V]) Keys() []K {
|
||||
// Values returns a slice of the values in the cache, from oldest to newest.
|
||||
// Expired entries are filtered out.
|
||||
func (c *cacheImpl[K, V]) Values() []V {
|
||||
values := make([]V, 0, len(c.items))
|
||||
now := time.Now()
|
||||
c.Lock()
|
||||
defer c.Unlock()
|
||||
values := make([]V, 0, len(c.items))
|
||||
for ent := c.evictList.Back(); ent != nil; ent = ent.Prev() {
|
||||
if !now.After(ent.Value.(*cacheItem[K, V]).expiresAt) {
|
||||
values = append(values, ent.Value.(*cacheItem[K, V]).value)
|
||||
|
||||
@@ -10,3 +10,7 @@
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Example binaries, produced by `go build` in the example modules
|
||||
/_example/application/application
|
||||
/_example/plugin/plugin
|
||||
|
||||
+76
-24
@@ -15,39 +15,55 @@ type Plugin struct {
|
||||
}
|
||||
|
||||
// create plugin (jrpc server) with NewServer where required param is a base url for rpc calls
|
||||
plugin := NewServer("/command")
|
||||
plugin := jrpc.NewServer("/command")
|
||||
|
||||
// then add you function to map
|
||||
plugin.Add("mycommand", func(id uint64, params json.RawMessage) Response {
|
||||
// then add your function to map
|
||||
plugin.Add("mycommand", func(id uint64, params json.RawMessage) jrpc.Response {
|
||||
return jrpc.EncodeResponse(id, "hello, it works", nil)
|
||||
})
|
||||
|
||||
// and run server with port number value
|
||||
plugin.Run(9090)
|
||||
plugin.Run(8080)
|
||||
```
|
||||
|
||||
The constructor `NewServer` accept two parameters:
|
||||
The constructor `NewServer` accepts two parameters:
|
||||
* `API` - a base url for rpc calls
|
||||
* `Options` - optional parameters such is timeouts, logger, limits, middlewares and etc.
|
||||
* `Auth` - set credentials basic auth to server, accepts `username` and `password`
|
||||
* `WithTimeout` - sets global timeouts for server requests, such as read, write and idle. Call accept `Timeouts` struct.
|
||||
* `WithLimits` - define limit for server call, accepts limit value in `float64` type
|
||||
* `WithThrottler` - sets throttler middleware with specify limit value
|
||||
* `WithtSignature` - sets server signature, accept appName, author and version. Disable by default.
|
||||
* `WithLogger` - define custom logger (e.g. [lgr](https://github.com/go-pkgz/lgr))
|
||||
* `WithMiddlewares` - sets custom middlewares list to server, accepts list of handler with idiomatic type `func(http.Handler) http.Handler`
|
||||
* `Options` - optional parameters such as timeouts, logger, limits, middlewares and so on.
|
||||
* `Auth` - sets basic auth credentials, accepts `username` and `password`. Auth is enforced only if both of them
|
||||
set to non-empty values; setting just one leaves the server serving every request unauthenticated
|
||||
* `WithTimeouts` - sets server timeouts, accepts a `Timeouts` struct with `ReadHeaderTimeout`, `WriteTimeout`,
|
||||
`IdleTimeout` and `CallTimeout`. `CallTimeout` limits the time allowed for a single call and responds with `503` if
|
||||
exceeded, and has to be set below `WriteTimeout`, otherwise the write deadline kills the connection before the
|
||||
`503` can be sent
|
||||
* `WithLimits` - defines a limit of calls/sec per client, accepts limit value in `float64` type
|
||||
* `WithThrottler` - sets throttler middleware limiting the number of parallel calls to the server
|
||||
* `WithSignature` - sets server signature, accepts appName, author and version. Disabled by default
|
||||
* `WithLogger` - defines custom logger (e.g. [lgr](https://github.com/go-pkgz/lgr))
|
||||
* `WithMiddlewares` - sets custom middlewares list to server, accepts list of handlers with idiomatic type `func(http.Handler) http.Handler`
|
||||
|
||||
Example with options:
|
||||
```go
|
||||
plugin := NewServer("/command",
|
||||
Auth("user", "password"),
|
||||
WithTimeout(Timeouts{ReadHeaderTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second}),
|
||||
WithThrottler(120),
|
||||
WithLimits(100),
|
||||
WithtSignature("the best plugin ever", "author", "1.0.0"),
|
||||
WithMiddlewares(middleware.Heartbeat('/ping'), middleware.Profiler, middleware.StripSlashes),
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/go-pkgz/jrpc"
|
||||
"github.com/go-pkgz/rest"
|
||||
)
|
||||
```
|
||||
|
||||
plugin := jrpc.NewServer("/command",
|
||||
jrpc.Auth("user", "password"),
|
||||
jrpc.WithTimeouts(jrpc.Timeouts{
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 10 * time.Second,
|
||||
CallTimeout: 25 * time.Second,
|
||||
}),
|
||||
jrpc.WithThrottler(120),
|
||||
jrpc.WithLimits(100),
|
||||
jrpc.WithSignature("the best plugin ever", "author", "1.0.0"),
|
||||
jrpc.WithMiddlewares(rest.Trace),
|
||||
)
|
||||
```
|
||||
|
||||
### Application (client)
|
||||
|
||||
@@ -67,8 +83,42 @@ if err = json.Unmarshal(*resp.Result, &message); err != nil {
|
||||
}
|
||||
```
|
||||
|
||||
*for functional examples for both plugin and application see [_example](https://github.com/go-pkgz/jrpc/tree/master/_example)*
|
||||
|
||||
### Running the example
|
||||
|
||||
[_example](https://github.com/go-pkgz/jrpc/tree/master/_example) has a working pair of a plugin and an application.
|
||||
Both are separate go modules pointing to the local jrpc with a `replace` directive, so no extra setup is needed
|
||||
beyond go 1.24 or later and a free local port 8080. Start the plugin first, in one terminal:
|
||||
|
||||
```sh
|
||||
cd _example/plugin
|
||||
go run .
|
||||
```
|
||||
|
||||
It registers two handlers and listens on port 8080:
|
||||
|
||||
```
|
||||
[INFO] add handler for store.save
|
||||
[INFO] add handler for store.load
|
||||
[INFO] listen on [::]:8080
|
||||
```
|
||||
|
||||
Then run the application in another terminal:
|
||||
|
||||
```sh
|
||||
cd _example/application
|
||||
go run .
|
||||
```
|
||||
|
||||
It calls the plugin three times and prints the results:
|
||||
|
||||
```
|
||||
stored {TS:2025-01-12 12:00:00 +0000 UTC Value:12345} with id=54118548792
|
||||
loaded {TS:2025-01-12 12:00:00 +0000 UTC Value:12345} from id=54118548792
|
||||
can't load for id=something, not found
|
||||
```
|
||||
|
||||
The application exits on its own, the plugin keeps listening until stopped with Ctrl-C.
|
||||
|
||||
## Technical details
|
||||
|
||||
* `jrpc.Server` runs on user-defined port as a regular http server
|
||||
@@ -95,7 +145,9 @@ if err = json.Unmarshal(*resp.Result, &message); err != nil {
|
||||
|
||||
* Params can be a struct, primitive type or slice of values, even with different types.
|
||||
* Server defines `ServerFn` handler function to react on a POST request. The handler provided by the user.
|
||||
* Communication between the server and the caller can be protected with basic auth.
|
||||
* Communication between the server and the caller can be protected with basic auth. The protection is on only if
|
||||
both user and password set with the `Auth` option; with either of them empty the server responds to every request
|
||||
without asking for credentials.
|
||||
* [Client](https://github.com/go-pkgz/jrpc/blob/master/client.go) provides a single method `Call` and return `Response`
|
||||
|
||||
<details><summary>response details:</summary>
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ type Client struct {
|
||||
// Empty args will be ignored, single arg will be marshaled as-us and multiple args marshaled as []interface{}.
|
||||
// Returns Response and error. Note: Response has it's own Error field, but that onw controlled by server.
|
||||
// Returned error represent client-level errors, like failed http call, failed marshaling and so on.
|
||||
func (r *Client) Call(method string, args ...interface{}) (*Response, error) {
|
||||
func (r *Client) Call(method string, args ...any) (*Response, error) {
|
||||
|
||||
var b []byte
|
||||
var err error
|
||||
|
||||
+4
-4
@@ -10,9 +10,9 @@ import (
|
||||
|
||||
// Request encloses method name and all params
|
||||
type Request struct {
|
||||
Method string `json:"method"` // method (function) name
|
||||
Params interface{} `json:"params,omitempty"` // function arguments
|
||||
ID uint64 `json:"id"` // unique call id
|
||||
Method string `json:"method"` // method (function) name
|
||||
Params any `json:"params,omitempty"` // function arguments
|
||||
ID uint64 `json:"id"` // unique call id
|
||||
}
|
||||
|
||||
// Response encloses result and error received from remote server
|
||||
@@ -23,7 +23,7 @@ type Response struct {
|
||||
}
|
||||
|
||||
// EncodeResponse convert anything (type interface{}) and incoming error (if any) to Response
|
||||
func EncodeResponse(id uint64, resp interface{}, e error) Response {
|
||||
func EncodeResponse(id uint64, resp any, e error) Response {
|
||||
v, err := json.Marshal(&resp)
|
||||
if err != nil {
|
||||
return Response{Error: err.Error()}
|
||||
|
||||
+3
-1
@@ -7,7 +7,9 @@ import (
|
||||
// Option func type
|
||||
type Option func(s *Server)
|
||||
|
||||
// Auth sets basic auth credentials, required
|
||||
// Auth sets basic auth credentials, optional.
|
||||
// Auth enforced only if both user and password set to non-empty values, otherwise the server
|
||||
// keeps serving every request unauthenticated. Setting just one of them doesn't enable auth.
|
||||
func Auth(user, password string) Option {
|
||||
return func(s *Server) {
|
||||
s.authUser = user
|
||||
|
||||
+46
-22
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -13,12 +14,13 @@ import (
|
||||
"github.com/go-pkgz/routegroup"
|
||||
)
|
||||
|
||||
// Server is json-rpc server with an optional basic auth
|
||||
// Server is json-rpc server with an optional basic auth.
|
||||
// Auth enforced only if both authUser and authPasswd set, see Auth option.
|
||||
type Server struct {
|
||||
api string // url path, i.e. "/command" or "/rpc" etc., required
|
||||
|
||||
authUser string // basic auth user name, should match Client.AuthUser, optional
|
||||
authPasswd string // basic auth password, should match Client.AuthPasswd, optional
|
||||
authUser string // basic auth user name, should match Client.AuthUser, optional, no auth if empty
|
||||
authPasswd string // basic auth password, should match Client.AuthPasswd, optional, no auth if empty
|
||||
customMiddlewares middlewares // list of custom middlewares, should match array of http.Handler func, optional
|
||||
|
||||
signature signaturePayload // add server signature to server response headers appName, author, version), disable by default
|
||||
@@ -81,15 +83,29 @@ func NewServer(api string, options ...Option) *Server {
|
||||
return srv
|
||||
}
|
||||
|
||||
// Run http server on given port
|
||||
// Run http server on given port, blocks until Shutdown called or the server failed
|
||||
func (s *Server) Run(port int) error {
|
||||
|
||||
if s.authUser == "" || s.authPasswd == "" {
|
||||
s.logger.Logf("[WARN] extension server runs without auth")
|
||||
if len(s.funcs.m) == 0 {
|
||||
return fmt.Errorf("nothing mapped for dispatch, Add has to be called prior to Run")
|
||||
}
|
||||
|
||||
if s.funcs.m == nil && len(s.funcs.m) == 0 {
|
||||
return fmt.Errorf("nothing mapped for dispatch, Add has to be called prior to Run")
|
||||
s.activate()
|
||||
|
||||
ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't listen on port %d: %w", port, err)
|
||||
}
|
||||
|
||||
return s.serve(ln)
|
||||
}
|
||||
|
||||
// activate makes http server with all the middlewares and the dispatch handler.
|
||||
// after this call Add won't accept new methods.
|
||||
func (s *Server) activate() {
|
||||
|
||||
if s.authUser == "" || s.authPasswd == "" {
|
||||
s.logger.Logf("[WARN] extension server runs without auth, both user and password have to be set to enable it")
|
||||
}
|
||||
|
||||
router := routegroup.New(http.NewServeMux())
|
||||
@@ -124,16 +140,26 @@ func (s *Server) Run(port int) error {
|
||||
|
||||
s.httpServer.Lock()
|
||||
s.httpServer.Server = &http.Server{
|
||||
Addr: fmt.Sprintf(":%d", port),
|
||||
Handler: router,
|
||||
ReadHeaderTimeout: s.timeouts.ReadHeaderTimeout,
|
||||
WriteTimeout: s.timeouts.WriteTimeout,
|
||||
IdleTimeout: s.timeouts.IdleTimeout,
|
||||
}
|
||||
s.httpServer.Unlock()
|
||||
}
|
||||
|
||||
s.logger.Logf("[INFO] listen on %d", port)
|
||||
return s.httpServer.ListenAndServe()
|
||||
// serve runs activated http server on the provided listener
|
||||
func (s *Server) serve(l net.Listener) error {
|
||||
s.httpServer.Lock()
|
||||
srv := s.httpServer.Server
|
||||
s.httpServer.Unlock()
|
||||
|
||||
if srv == nil {
|
||||
return fmt.Errorf("server is not activated")
|
||||
}
|
||||
|
||||
s.logger.Logf("[INFO] listen on %s", l.Addr())
|
||||
return srv.Serve(l)
|
||||
}
|
||||
|
||||
// Shutdown http server
|
||||
@@ -202,7 +228,8 @@ func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
|
||||
rest.RenderJSON(w, fn(req.ID, params))
|
||||
}
|
||||
|
||||
// basicAuth middleware. enabled only if both AuthUser and AuthPasswd defined.
|
||||
// basicAuth middleware, enabled only if both authUser and authPasswd set to non-empty values.
|
||||
// with either of them empty every request passes through unauthenticated.
|
||||
func (s *Server) basicAuth(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -229,30 +256,27 @@ func getDefaultTimeouts() Timeouts {
|
||||
}
|
||||
}
|
||||
|
||||
// timeout middleware cancels context after given duration
|
||||
// timeout middleware limits the time allowed for the call, responds with 503 and drops
|
||||
// the late handler writes if the deadline reached
|
||||
func timeout(dt time.Duration) func(http.Handler) http.Handler {
|
||||
return func(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), dt)
|
||||
defer cancel()
|
||||
h.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
return http.TimeoutHandler(h, dt, `{"error":"call timeout"}`)
|
||||
}
|
||||
}
|
||||
|
||||
// L defined logger interface used for an optional rest logging
|
||||
type L interface {
|
||||
Logf(format string, args ...interface{})
|
||||
Logf(format string, args ...any)
|
||||
}
|
||||
|
||||
// LoggerFunc type is an adapter to allow the use of ordinary functions as Logger.
|
||||
type LoggerFunc func(format string, args ...interface{})
|
||||
type LoggerFunc func(format string, args ...any)
|
||||
|
||||
// Logf calls f(id)
|
||||
func (f LoggerFunc) Logf(format string, args ...interface{}) { f(format, args...) }
|
||||
func (f LoggerFunc) Logf(format string, args ...any) { f(format, args...) }
|
||||
|
||||
// NoOpLogger logger does nothing
|
||||
var NoOpLogger = LoggerFunc(func(format string, args ...interface{}) {}) //nolint
|
||||
var NoOpLogger = LoggerFunc(func(format string, args ...any) {}) //nolint
|
||||
|
||||
// rateLimitByIP returns middleware that limits requests per second for each client IP.
|
||||
// Uses X-Real-IP header (set by rest.RealIP middleware) for client identification.
|
||||
|
||||
+16
-1
@@ -11,11 +11,26 @@ import (
|
||||
)
|
||||
|
||||
// Sizer allows to perform size-based restrictions, optional.
|
||||
// If not defined both maxValueSize and maxCacheSize checks will be ignored
|
||||
// Values implementing it define their own size, []byte and string are sized by length.
|
||||
// For any other type both maxValueSize and maxCacheSize checks are ignored.
|
||||
type Sizer interface {
|
||||
Size() int
|
||||
}
|
||||
|
||||
// sizeOf returns the size of the value for size-based restrictions and reports
|
||||
// whether the value can be sized at all. Sizer takes priority over the built-in types.
|
||||
func sizeOf(value any) (int, bool) {
|
||||
switch v := value.(type) {
|
||||
case Sizer:
|
||||
return v.Size(), true
|
||||
case []byte:
|
||||
return len(v), true
|
||||
case string:
|
||||
return len(v), true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// LoadingCache defines guava-like cache with Get method returning cached value ao retrieving it if not in cache
|
||||
type LoadingCache[V any] interface {
|
||||
Get(key string, fn func() (V, error)) (val V, err error) // load or get from cache
|
||||
|
||||
+5
-5
@@ -2,11 +2,11 @@ package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
@@ -67,12 +67,12 @@ func (m *RedisPubSub) Publish(fromID, key string) error {
|
||||
func (m *RedisPubSub) Close() error {
|
||||
close(m.done)
|
||||
|
||||
errs := new(multierror.Error)
|
||||
var errs []error
|
||||
if err := m.pubSub.Close(); err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("problem closing pubSub client: %w", err))
|
||||
errs = append(errs, fmt.Errorf("problem closing pubSub client: %w", err))
|
||||
}
|
||||
if err := m.client.Close(); err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("problem closing redis client: %w", err))
|
||||
errs = append(errs, fmt.Errorf("problem closing redis client: %w", err))
|
||||
}
|
||||
return errs.ErrorOrNil()
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
+34
-20
@@ -18,9 +18,14 @@ type ExpirableCache[V any] struct {
|
||||
currentSize int64
|
||||
id string
|
||||
backend *expirable.LRU[string, V]
|
||||
loads loadGroup[V]
|
||||
}
|
||||
|
||||
// NewExpirableCache makes expirable LoadingCache implementation, 1000 max keys by default and 5m TTL
|
||||
// NewExpirableCache makes expirable LoadingCache implementation, 1000 max keys by default and 5m TTL.
|
||||
// Note that the underlying hashicorp/golang-lru expirable backend calls eviction callbacks while
|
||||
// holding its own lock, so an OnEvicted handler must not call back into the same cache, it would deadlock.
|
||||
// The same handler is safe to use with LruCache and with the v1 ExpirableCache.
|
||||
// Reported upstream as https://github.com/hashicorp/golang-lru/issues/230
|
||||
func NewExpirableCache[V any](opts ...Option[V]) (*ExpirableCache[V], error) {
|
||||
res := ExpirableCache[V]{
|
||||
Workers: Workers[V]{
|
||||
@@ -46,8 +51,7 @@ func NewExpirableCache[V any](opts ...Option[V]) (*ExpirableCache[V], error) {
|
||||
if res.onEvicted != nil {
|
||||
res.onEvicted(key, value)
|
||||
}
|
||||
if s, ok := any(value).(Sizer); ok {
|
||||
size := s.Size()
|
||||
if size, ok := sizeOf(value); ok {
|
||||
atomic.AddInt64(&res.currentSize, -1*int64(size))
|
||||
}
|
||||
// ignore the error on Publish as we don't have log inside the module and
|
||||
@@ -66,26 +70,36 @@ func (c *ExpirableCache[V]) Get(key string, fn func() (V, error)) (data V, err e
|
||||
return v, nil
|
||||
}
|
||||
|
||||
if data, err = fn(); err != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, err
|
||||
}
|
||||
atomic.AddInt64(&c.Misses, 1)
|
||||
// concurrent callers for the same key wait for the first load instead of loading on their own,
|
||||
// otherwise each of them would add the value and count its size again
|
||||
return c.loads.do(key, func() (V, error) {
|
||||
if v, ok := c.backend.Get(key); ok { // filled by the load we were waiting for
|
||||
atomic.AddInt64(&c.Hits, 1)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
if !c.allowed(key, data) {
|
||||
return data, nil
|
||||
}
|
||||
data, err := fn()
|
||||
if err != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, err
|
||||
}
|
||||
atomic.AddInt64(&c.Misses, 1)
|
||||
|
||||
if s, ok := any(data).(Sizer); ok {
|
||||
if c.maxCacheSize > 0 && atomic.LoadInt64(&c.currentSize)+int64(s.Size()) >= c.maxCacheSize {
|
||||
if !c.allowed(key, data) {
|
||||
return data, nil
|
||||
}
|
||||
atomic.AddInt64(&c.currentSize, int64(s.Size()))
|
||||
}
|
||||
|
||||
c.backend.Add(key, data)
|
||||
if size, ok := sizeOf(data); ok {
|
||||
if c.maxCacheSize > 0 && atomic.LoadInt64(&c.currentSize)+int64(size) >= c.maxCacheSize {
|
||||
return data, nil
|
||||
}
|
||||
atomic.AddInt64(&c.currentSize, int64(size))
|
||||
}
|
||||
|
||||
return data, nil
|
||||
c.backend.Add(key, data)
|
||||
|
||||
return data, nil
|
||||
})
|
||||
}
|
||||
|
||||
// Invalidate removes keys with passed predicate fn, i.e. fn(key) should be true to get evicted
|
||||
@@ -154,14 +168,14 @@ func (c *ExpirableCache[V]) keys() int {
|
||||
}
|
||||
|
||||
func (c *ExpirableCache[V]) allowed(key string, data V) bool {
|
||||
if c.backend.Len() >= c.maxKeys {
|
||||
if c.maxKeys > 0 && c.backend.Len() >= c.maxKeys {
|
||||
return false
|
||||
}
|
||||
if c.maxKeySize > 0 && len(key) > c.maxKeySize {
|
||||
return false
|
||||
}
|
||||
if s, ok := any(data).(Sizer); ok {
|
||||
if c.maxValueSize > 0 && s.Size() >= c.maxValueSize {
|
||||
if size, ok := sizeOf(data); ok {
|
||||
if c.maxValueSize > 0 && size >= c.maxValueSize {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
package lcw
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ErrLoaderPanic is what callers waiting for a load get when the loader panicked.
|
||||
// The panic value itself keeps propagating in the goroutine that ran the loader.
|
||||
var ErrLoaderPanic = errors.New("cache loader panic")
|
||||
|
||||
// loadGroup makes sure only one load function per key runs at a time.
|
||||
// concurrent calls for the same key wait for the in-flight one and share its result,
|
||||
// so a cold key is loaded once instead of once per caller.
|
||||
// the loader must not call Get for the same key on the same cache, it would wait for itself.
|
||||
type loadGroup[V any] struct {
|
||||
mu sync.Mutex
|
||||
calls map[string]*loadCall[V]
|
||||
}
|
||||
|
||||
type loadCall[V any] struct {
|
||||
wg sync.WaitGroup
|
||||
val V
|
||||
err error
|
||||
}
|
||||
|
||||
// do calls fn for the given key unless the same key is already being loaded,
|
||||
// in that case it waits for the in-flight load and returns its result.
|
||||
func (g *loadGroup[V]) do(key string, fn func() (V, error)) (V, error) {
|
||||
g.mu.Lock()
|
||||
if g.calls == nil {
|
||||
g.calls = make(map[string]*loadCall[V])
|
||||
}
|
||||
if call, ok := g.calls[key]; ok {
|
||||
g.mu.Unlock()
|
||||
call.wg.Wait()
|
||||
return call.val, call.err
|
||||
}
|
||||
|
||||
call := &loadCall[V]{}
|
||||
call.wg.Add(1)
|
||||
g.calls[key] = call
|
||||
g.mu.Unlock()
|
||||
|
||||
// waiters are released even if fn panics, and get an error instead of a zero value
|
||||
// with no error at all. The panic keeps propagating in the calling goroutine, as it
|
||||
// would without the load coordination.
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
// a fixed error, formatting the panic value here would run arbitrary
|
||||
// code of its Error or String method before the waiters are released
|
||||
call.err = ErrLoaderPanic
|
||||
g.done(key, call)
|
||||
panic(p)
|
||||
}
|
||||
g.done(key, call)
|
||||
}()
|
||||
|
||||
call.val, call.err = fn()
|
||||
return call.val, call.err
|
||||
}
|
||||
|
||||
// done drops the in-flight call and releases everybody waiting for it
|
||||
func (g *loadGroup[V]) done(key string, call *loadCall[V]) {
|
||||
g.mu.Lock()
|
||||
delete(g.calls, key)
|
||||
g.mu.Unlock()
|
||||
call.wg.Done()
|
||||
}
|
||||
+36
-21
@@ -2,6 +2,7 @@ package lcw
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -17,6 +18,7 @@ type LruCache[V any] struct {
|
||||
backend *lru.Cache[string, V]
|
||||
currentSize int64
|
||||
id string // uuid identifying cache instance
|
||||
loads loadGroup[V]
|
||||
}
|
||||
|
||||
// NewLruCache makes LRU LoadingCache implementation, 1000 max keys by default
|
||||
@@ -48,8 +50,7 @@ func (c *LruCache[V]) init() error {
|
||||
if c.onEvicted != nil {
|
||||
c.onEvicted(key, value)
|
||||
}
|
||||
if s, ok := any(value).(Sizer); ok {
|
||||
size := s.Size()
|
||||
if size, ok := sizeOf(value); ok {
|
||||
atomic.AddInt64(&c.currentSize, -1*int64(size))
|
||||
}
|
||||
_ = c.eventBus.Publish(c.id, key) // signal invalidation to other nodes
|
||||
@@ -57,7 +58,11 @@ func (c *LruCache[V]) init() error {
|
||||
|
||||
var err error
|
||||
// OnEvicted called automatically for expired and manually deleted
|
||||
if c.backend, err = lru.NewWithEvict[string, V](c.maxKeys, onEvicted); err != nil {
|
||||
maxKeys := c.maxKeys
|
||||
if maxKeys <= 0 { // 0 means unlimited, lru backend requires a positive size
|
||||
maxKeys = math.MaxInt
|
||||
}
|
||||
if c.backend, err = lru.NewWithEvict[string, V](maxKeys, onEvicted); err != nil {
|
||||
return fmt.Errorf("failed to make lru cache backend: %w", err)
|
||||
}
|
||||
|
||||
@@ -71,29 +76,39 @@ func (c *LruCache[V]) Get(key string, fn func() (V, error)) (data V, err error)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
if data, err = fn(); err != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, err
|
||||
}
|
||||
// concurrent callers for the same key wait for the first load instead of loading on their own,
|
||||
// otherwise each of them would add the value and count its size again
|
||||
return c.loads.do(key, func() (V, error) {
|
||||
if v, ok := c.backend.Get(key); ok { // filled by the load we were waiting for
|
||||
atomic.AddInt64(&c.Hits, 1)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
atomic.AddInt64(&c.Misses, 1)
|
||||
data, err := fn()
|
||||
if err != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, err
|
||||
}
|
||||
|
||||
if !c.allowed(key, data) {
|
||||
return data, nil
|
||||
}
|
||||
atomic.AddInt64(&c.Misses, 1)
|
||||
|
||||
c.backend.Add(key, data)
|
||||
if !c.allowed(key, data) {
|
||||
return data, nil
|
||||
}
|
||||
|
||||
if s, ok := any(data).(Sizer); ok {
|
||||
atomic.AddInt64(&c.currentSize, int64(s.Size()))
|
||||
if c.maxCacheSize > 0 && atomic.LoadInt64(&c.currentSize) > c.maxCacheSize {
|
||||
for atomic.LoadInt64(&c.currentSize) > c.maxCacheSize {
|
||||
c.backend.RemoveOldest()
|
||||
c.backend.Add(key, data)
|
||||
|
||||
if size, ok := sizeOf(data); ok {
|
||||
atomic.AddInt64(&c.currentSize, int64(size))
|
||||
for c.maxCacheSize > 0 && atomic.LoadInt64(&c.currentSize) > c.maxCacheSize {
|
||||
if _, _, ok := c.backend.RemoveOldest(); !ok { // nothing left to evict
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data, nil
|
||||
return data, nil
|
||||
})
|
||||
}
|
||||
|
||||
// Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key.
|
||||
@@ -161,8 +176,8 @@ func (c *LruCache[V]) allowed(key string, data V) bool {
|
||||
if c.maxKeySize > 0 && len(key) > c.maxKeySize {
|
||||
return false
|
||||
}
|
||||
if s, ok := any(data).(Sizer); ok {
|
||||
if c.maxValueSize > 0 && s.Size() >= c.maxValueSize {
|
||||
if size, ok := sizeOf(data); ok {
|
||||
if c.maxValueSize > 0 && size >= c.maxValueSize {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
+34
-14
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/go-pkgz/lcw/v2/eventbus"
|
||||
)
|
||||
|
||||
// Workers holds cache configuration options
|
||||
type Workers[V any] struct {
|
||||
maxKeys int
|
||||
maxValueSize int
|
||||
@@ -16,6 +17,8 @@ type Workers[V any] struct {
|
||||
onEvicted func(key string, value V)
|
||||
eventBus eventbus.PubSub
|
||||
strToV func(string) V
|
||||
|
||||
redisKeyPrefix string
|
||||
}
|
||||
|
||||
// Option func type
|
||||
@@ -29,50 +32,53 @@ func NewOpts[T any]() *WorkerOptions[T] {
|
||||
return &WorkerOptions[T]{}
|
||||
}
|
||||
|
||||
// MaxValSize functional option defines the largest value's size allowed to be cached
|
||||
// MaxValSize functional option defines the largest value's size allowed to be cached.
|
||||
// Applies to values implementing Sizer as well as []byte and string, other types are not limited.
|
||||
// By default it is 0, which means unlimited.
|
||||
func (o *WorkerOptions[V]) MaxValSize(max int) Option[V] {
|
||||
func (o *WorkerOptions[V]) MaxValSize(maximum int) Option[V] {
|
||||
return func(o *Workers[V]) error {
|
||||
if max < 0 {
|
||||
if maximum < 0 {
|
||||
return fmt.Errorf("negative max value size")
|
||||
}
|
||||
o.maxValueSize = max
|
||||
o.maxValueSize = maximum
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MaxKeySize functional option defines the largest key's size allowed to be used in cache
|
||||
// By default it is 0, which means unlimited.
|
||||
func (o *WorkerOptions[V]) MaxKeySize(max int) Option[V] {
|
||||
func (o *WorkerOptions[V]) MaxKeySize(maximum int) Option[V] {
|
||||
return func(o *Workers[V]) error {
|
||||
if max < 0 {
|
||||
if maximum < 0 {
|
||||
return fmt.Errorf("negative max key size")
|
||||
}
|
||||
o.maxKeySize = max
|
||||
o.maxKeySize = maximum
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MaxKeys functional option defines how many keys to keep.
|
||||
// By default, it is 0, which means unlimited.
|
||||
func (o *WorkerOptions[V]) MaxKeys(max int) Option[V] {
|
||||
func (o *WorkerOptions[V]) MaxKeys(maximum int) Option[V] {
|
||||
return func(o *Workers[V]) error {
|
||||
if max < 0 {
|
||||
if maximum < 0 {
|
||||
return fmt.Errorf("negative max keys")
|
||||
}
|
||||
o.maxKeys = max
|
||||
o.maxKeys = maximum
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MaxCacheSize functional option defines the total size of cached data.
|
||||
// Applies to values implementing Sizer as well as []byte and string, other types are not counted.
|
||||
// Not supported by RedisCache, which accepts the option but ignores it.
|
||||
// By default, it is 0, which means unlimited.
|
||||
func (o *WorkerOptions[V]) MaxCacheSize(max int64) Option[V] {
|
||||
func (o *WorkerOptions[V]) MaxCacheSize(maximum int64) Option[V] {
|
||||
return func(o *Workers[V]) error {
|
||||
if max < 0 {
|
||||
if maximum < 0 {
|
||||
return fmt.Errorf("negative max cache size")
|
||||
}
|
||||
o.maxCacheSize = max
|
||||
o.maxCacheSize = maximum
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -89,7 +95,9 @@ func (o *WorkerOptions[V]) TTL(ttl time.Duration) Option[V] {
|
||||
}
|
||||
}
|
||||
|
||||
// OnEvicted sets callback on invalidation event
|
||||
// OnEvicted sets callback on invalidation event.
|
||||
// The callback runs outside the cache lock for LruCache, but ExpirableCache calls it while its
|
||||
// backend lock is held, so with ExpirableCache the handler must not call back into the same cache.
|
||||
func (o *WorkerOptions[V]) OnEvicted(fn func(key string, value V)) Option[V] {
|
||||
return func(o *Workers[V]) error {
|
||||
o.onEvicted = fn
|
||||
@@ -105,6 +113,18 @@ func (o *WorkerOptions[V]) EventBus(pubSub eventbus.PubSub) Option[V] {
|
||||
}
|
||||
}
|
||||
|
||||
// RedisKeyPrefix sets the prefix for all keys stored in redis, making the cache
|
||||
// use only its own namespace instead of the whole redis database.
|
||||
// Applies to RedisCache only, ignored by other backends. Empty by default, which means
|
||||
// the cache assumes exclusive ownership of the selected redis database.
|
||||
// Note that with a prefix set and MaxKeys defined, key counting scans the keyspace on every miss.
|
||||
func (o *WorkerOptions[V]) RedisKeyPrefix(prefix string) Option[V] {
|
||||
return func(o *Workers[V]) error {
|
||||
o.redisKeyPrefix = prefix
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// StrToV sets strToV function for RedisCache
|
||||
func (o *WorkerOptions[V]) StrToV(fn func(string) V) Option[V] {
|
||||
return func(o *Workers[V]) error {
|
||||
|
||||
+96
-47
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -18,12 +19,16 @@ const RedisValueSizeLimit = 512 * 1024 * 1024
|
||||
type RedisCache[V any] struct {
|
||||
Workers[V]
|
||||
CacheStat
|
||||
backend *redis.Client
|
||||
backend redis.UniversalClient
|
||||
loads loadGroup[V]
|
||||
}
|
||||
|
||||
// NewRedisCache makes Redis LoadingCache implementation.
|
||||
// Supports only string and string-based types and will return error otherwise.
|
||||
func NewRedisCache[V any](backend *redis.Client, opts ...Option[V]) (*RedisCache[V], error) {
|
||||
// Without RedisKeyPrefix the cache assumes exclusive ownership of the selected Redis database,
|
||||
// i.e. Purge flushes it entirely and Keys, Stat and MaxKeys count every key in it.
|
||||
// MaxCacheSize is not supported by this backend, it is accepted but ignored, and Stat reports size 0.
|
||||
func NewRedisCache[V any](backend redis.UniversalClient, opts ...Option[V]) (*RedisCache[V], error) {
|
||||
// check if V is string, not underlying type but directly, and otherwise return error if strToV is nil as it should be defined
|
||||
|
||||
res := RedisCache[V]{
|
||||
@@ -60,54 +65,78 @@ func NewRedisCache[V any](backend *redis.Client, opts ...Option[V]) (*RedisCache
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
// fullKey makes the physical redis key for the given logical key
|
||||
func (c *RedisCache[V]) fullKey(key string) string {
|
||||
return c.redisKeyPrefix + key
|
||||
}
|
||||
|
||||
// scanKeys returns all physical keys belonging to this cache
|
||||
func (c *RedisCache[V]) scanKeys() []string {
|
||||
return c.backend.Keys(context.Background(), escapeGlob(c.redisKeyPrefix)+"*").Val()
|
||||
}
|
||||
|
||||
// Get gets value by key or load with fn if not found in cache
|
||||
func (c *RedisCache[V]) Get(key string, fn func() (V, error)) (data V, err error) {
|
||||
v, getErr := c.backend.Get(context.Background(), key).Result()
|
||||
v, getErr := c.backend.Get(context.Background(), c.fullKey(key)).Result()
|
||||
switch {
|
||||
// RedisClient returns nil when find a key in DB
|
||||
case getErr == nil:
|
||||
atomic.AddInt64(&c.Hits, 1)
|
||||
switch any(data).(type) {
|
||||
case string:
|
||||
return any(v).(V), nil
|
||||
default:
|
||||
return c.strToV(v), nil
|
||||
}
|
||||
// RedisClient returns redis.Nil when doesn't find a key in DB
|
||||
return c.toV(v), nil
|
||||
// RedisClient returns redis.Nil when doesn't find a key in DB, load it below
|
||||
case errors.Is(getErr, redis.Nil):
|
||||
if data, err = fn(); err != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, err
|
||||
}
|
||||
// RedisClient returns !nil when something goes wrong while get data
|
||||
default:
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
switch any(data).(type) {
|
||||
case string:
|
||||
return any(v).(V), getErr
|
||||
default:
|
||||
return c.strToV(v), getErr
|
||||
return c.toV(v), getErr
|
||||
}
|
||||
|
||||
// concurrent callers for the same key wait for the first load instead of loading on their own
|
||||
return c.loads.do(key, func() (V, error) {
|
||||
cached, cachedErr := c.backend.Get(context.Background(), c.fullKey(key)).Result()
|
||||
switch {
|
||||
case cachedErr == nil:
|
||||
atomic.AddInt64(&c.Hits, 1) // filled by the load we were waiting for
|
||||
return c.toV(cached), nil
|
||||
case !errors.Is(cachedErr, redis.Nil): // a broken backend is not a miss, same as above
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
var empty V // cached holds no value here, no point running StrToV over it
|
||||
return empty, cachedErr
|
||||
}
|
||||
|
||||
data, err := fn()
|
||||
if err != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, err
|
||||
}
|
||||
atomic.AddInt64(&c.Misses, 1)
|
||||
|
||||
if !c.allowed(key, data) {
|
||||
return data, nil
|
||||
}
|
||||
|
||||
if _, setErr := c.backend.Set(context.Background(), c.fullKey(key), data, c.ttl).Result(); setErr != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, setErr
|
||||
}
|
||||
}
|
||||
atomic.AddInt64(&c.Misses, 1)
|
||||
|
||||
if !c.allowed(key, data) {
|
||||
return data, nil
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
_, setErr := c.backend.Set(context.Background(), key, data, c.ttl).Result()
|
||||
if setErr != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, setErr
|
||||
// toV converts the string stored in redis back to V, directly for string and via StrToV for string-based types
|
||||
func (c *RedisCache[V]) toV(s string) V {
|
||||
var v V
|
||||
if _, ok := any(v).(string); ok {
|
||||
return any(s).(V)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
return c.strToV(s)
|
||||
}
|
||||
|
||||
// Invalidate removes keys with passed predicate fn, i.e. fn(key) should be true to get evicted
|
||||
func (c *RedisCache[V]) Invalidate(fn func(key string) bool) {
|
||||
for _, key := range c.backend.Keys(context.Background(), "*").Val() { // Keys() returns copy of cache's key, safe to remove directly
|
||||
if fn(key) {
|
||||
for _, key := range c.scanKeys() { // Keys() returns copy of cache's key, safe to remove directly
|
||||
if fn(strings.TrimPrefix(key, c.redisKeyPrefix)) {
|
||||
c.backend.Del(context.Background(), key)
|
||||
}
|
||||
}
|
||||
@@ -115,33 +144,43 @@ func (c *RedisCache[V]) Invalidate(fn func(key string) bool) {
|
||||
|
||||
// Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key.
|
||||
func (c *RedisCache[V]) Peek(key string) (data V, found bool) {
|
||||
ret, err := c.backend.Get(context.Background(), key).Result()
|
||||
ret, err := c.backend.Get(context.Background(), c.fullKey(key)).Result()
|
||||
if err != nil {
|
||||
var emptyValue V
|
||||
return emptyValue, false
|
||||
}
|
||||
switch any(data).(type) {
|
||||
case string:
|
||||
return any(ret).(V), true
|
||||
default:
|
||||
return any(ret).(V), true
|
||||
}
|
||||
return c.toV(ret), true
|
||||
}
|
||||
|
||||
// Purge clears the cache completely.
|
||||
// Purge clears the cache completely. Without RedisKeyPrefix set it flushes the whole redis database.
|
||||
func (c *RedisCache[V]) Purge() {
|
||||
c.backend.FlushDB(context.Background())
|
||||
|
||||
if c.redisKeyPrefix == "" {
|
||||
c.backend.FlushDB(context.Background())
|
||||
return
|
||||
}
|
||||
// deleted one by one, a multi-key Del is routed by the first key's slot
|
||||
// and fails across slots on a cluster client
|
||||
for _, key := range c.scanKeys() {
|
||||
c.backend.Del(context.Background(), key)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete cache item by key
|
||||
func (c *RedisCache[V]) Delete(key string) {
|
||||
c.backend.Del(context.Background(), key)
|
||||
c.backend.Del(context.Background(), c.fullKey(key))
|
||||
}
|
||||
|
||||
// Keys gets all keys for the cache
|
||||
func (c *RedisCache[V]) Keys() (res []string) {
|
||||
return c.backend.Keys(context.Background(), "*").Val()
|
||||
keys := c.scanKeys()
|
||||
if c.redisKeyPrefix == "" {
|
||||
return keys
|
||||
}
|
||||
res = make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
res = append(res, strings.TrimPrefix(key, c.redisKeyPrefix))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Stat returns cache statistics
|
||||
@@ -165,20 +204,30 @@ func (c *RedisCache[V]) size() int64 {
|
||||
}
|
||||
|
||||
func (c *RedisCache[V]) keys() int {
|
||||
return int(c.backend.DBSize(context.Background()).Val())
|
||||
if c.redisKeyPrefix == "" {
|
||||
return int(c.backend.DBSize(context.Background()).Val())
|
||||
}
|
||||
return len(c.scanKeys())
|
||||
}
|
||||
|
||||
func (c *RedisCache[V]) allowed(key string, data V) bool {
|
||||
if c.maxKeys > 0 && c.backend.DBSize(context.Background()).Val() >= int64(c.maxKeys) {
|
||||
if c.maxKeys > 0 && c.keys() >= c.maxKeys {
|
||||
return false
|
||||
}
|
||||
if c.maxKeySize > 0 && len(key) > c.maxKeySize {
|
||||
return false
|
||||
}
|
||||
if s, ok := any(data).(Sizer); ok {
|
||||
if c.maxValueSize > 0 && (s.Size() >= c.maxValueSize) {
|
||||
if size, ok := sizeOf(data); ok {
|
||||
if c.maxValueSize > 0 && size >= c.maxValueSize {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// escapeGlob escapes redis glob-style pattern metacharacters, so a key prefix
|
||||
// containing them still matches literally
|
||||
func escapeGlob(s string) string {
|
||||
replacer := strings.NewReplacer(`\`, `\\`, "*", `\*`, "?", `\?`, "[", `\[`, "]", `\]`)
|
||||
return replacer.Replace(s)
|
||||
}
|
||||
|
||||
+12
-11
@@ -2,6 +2,7 @@ package lcw
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -35,24 +36,24 @@ func (m *Scache[V]) Close() error {
|
||||
return m.lc.Close()
|
||||
}
|
||||
|
||||
// Flush clears cache and calls postFlushFn async
|
||||
// Flush clears keys of the requested partition, matching the requested scopes.
|
||||
// With no scopes set every key of the partition is removed, keys of other partitions are kept.
|
||||
func (m *Scache[V]) Flush(req FlusherRequest) {
|
||||
if len(req.scopes) == 0 {
|
||||
m.lc.Purge()
|
||||
return
|
||||
}
|
||||
|
||||
// check if fullKey has matching scopes
|
||||
// check if fullKey belongs to the requested partition and has matching scopes
|
||||
inScope := func(fullKey string) bool {
|
||||
key, err := parseKey(fullKey)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if key.partition != req.partition {
|
||||
return false
|
||||
}
|
||||
if len(req.scopes) == 0 { // no scopes means the whole partition
|
||||
return true
|
||||
}
|
||||
for _, s := range req.scopes {
|
||||
for _, ks := range key.scopes {
|
||||
if ks == s {
|
||||
return true
|
||||
}
|
||||
if slices.Contains(key.scopes, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
+13
-9
@@ -1,18 +1,18 @@
|
||||
package lcw
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// New parses uri and makes any of supported caches
|
||||
// supported URIs:
|
||||
// - redis://<ip>:<port>?db=123&max_keys=10
|
||||
// - redis://<ip>:<port>?db=123&max_keys=10&redis_key_prefix=lcw:
|
||||
// - mem://lru?max_keys=10&max_cache_size=1024
|
||||
// - mem://expirable?ttl=30s&max_val_size=100
|
||||
// - nop://
|
||||
@@ -55,13 +55,13 @@ func New[V any](uri string) (LoadingCache[V], error) {
|
||||
}
|
||||
|
||||
func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) {
|
||||
errs := new(multierror.Error)
|
||||
var errs []error
|
||||
o := NewOpts[V]()
|
||||
|
||||
if v := q.Get("max_val_size"); v != "" {
|
||||
vv, e := strconv.Atoi(v)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("max_val_size query param %s: %w", v, e))
|
||||
errs = append(errs, fmt.Errorf("max_val_size query param %s: %w", v, e))
|
||||
} else {
|
||||
opts = append(opts, o.MaxValSize(vv))
|
||||
}
|
||||
@@ -70,7 +70,7 @@ func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) {
|
||||
if v := q.Get("max_key_size"); v != "" {
|
||||
vv, e := strconv.Atoi(v)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("max_key_size query param %s: %w", v, e))
|
||||
errs = append(errs, fmt.Errorf("max_key_size query param %s: %w", v, e))
|
||||
} else {
|
||||
opts = append(opts, o.MaxKeySize(vv))
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) {
|
||||
if v := q.Get("max_keys"); v != "" {
|
||||
vv, e := strconv.Atoi(v)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("max_keys query param %s: %w", v, e))
|
||||
errs = append(errs, fmt.Errorf("max_keys query param %s: %w", v, e))
|
||||
} else {
|
||||
opts = append(opts, o.MaxKeys(vv))
|
||||
}
|
||||
@@ -88,7 +88,7 @@ func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) {
|
||||
if v := q.Get("max_cache_size"); v != "" {
|
||||
vv, e := strconv.ParseInt(v, 10, 64)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("max_cache_size query param %s: %w", v, e))
|
||||
errs = append(errs, fmt.Errorf("max_cache_size query param %s: %w", v, e))
|
||||
} else {
|
||||
opts = append(opts, o.MaxCacheSize(vv))
|
||||
}
|
||||
@@ -97,13 +97,17 @@ func optionsFromQuery[V any](q url.Values) (opts []Option[V], err error) {
|
||||
if v := q.Get("ttl"); v != "" {
|
||||
vv, e := time.ParseDuration(v)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("ttl query param %s: %w", v, e))
|
||||
errs = append(errs, fmt.Errorf("ttl query param %s: %w", v, e))
|
||||
} else {
|
||||
opts = append(opts, o.TTL(vv))
|
||||
}
|
||||
}
|
||||
|
||||
return opts, errs.ErrorOrNil()
|
||||
if v := q.Get("redis_key_prefix"); v != "" {
|
||||
opts = append(opts, o.RedisKeyPrefix(v))
|
||||
}
|
||||
|
||||
return opts, errors.Join(errs...)
|
||||
}
|
||||
|
||||
func redisOptionsFromURL(u *url.URL) (*redis.Options, error) {
|
||||
|
||||
+4
@@ -44,6 +44,10 @@ linters:
|
||||
- linters:
|
||||
- gosec
|
||||
text: 'G114: Use of net/http serve function that has no support for setting timeouts'
|
||||
- linters:
|
||||
- gosec
|
||||
path: _test\.go$
|
||||
text: 'G705: XSS via taint analysis'
|
||||
- linters:
|
||||
- revive
|
||||
- unparam
|
||||
|
||||
+6
-6
@@ -6,7 +6,7 @@
|
||||
## Features
|
||||
|
||||
- Simple and intuitive API for route grouping and route mounting.
|
||||
- Lightweight, just about 100 LOC
|
||||
- Lightweight, contained in a single file
|
||||
- Easy middleware integration for individual routes or groups of routes.
|
||||
- Seamless integration with Go's standard `http.ServeMux`.
|
||||
- Fully compatible with the `http.Handler` interface and can be used as a drop-in replacement for `http.ServeMux`.
|
||||
@@ -133,7 +133,7 @@ router.Group().Route(func(b *routegroup.Bundle) {
|
||||
|
||||
**Setting optional `NotFoundHandler`**
|
||||
|
||||
It is possible to set a custom `NotFoundHandler` for the group. This handler will be called when no other route matches the request:
|
||||
Set a custom `NotFoundHandler` for the group. It runs only when `http.ServeMux` would return 404:
|
||||
|
||||
```go
|
||||
group.NotFoundHandler(func(w http.ResponseWriter, _ *http.Request) {
|
||||
@@ -141,11 +141,11 @@ group.NotFoundHandler(func(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
```
|
||||
|
||||
If a custom `NotFoundHandler` is not configured, `routegroup` will default to using the standard library behavior.
|
||||
Without a custom `NotFoundHandler`, `routegroup` uses the standard library behavior.
|
||||
|
||||
Note on 405: In the current design, `routegroup` applies root-level middlewares to all requests at the top level without installing a catch‑all route. This preserves native `405 Method Not Allowed` responses from `http.ServeMux` when a path exists but a wrong method is used. A configured `NotFoundHandler` is only invoked when no route matches; it does not interfere with 405 handling. The custom `NotFoundHandler` will have the root bundle's global middlewares applied to it.
|
||||
Native mux responses: `routegroup` applies root-level middlewares to all requests without installing a catch‑all route. A configured `NotFoundHandler` runs only when `http.ServeMux` would return 404. Method mismatches return `405 Method Not Allowed` with an `Allow` header, and `http.ServeMux` performs path-cleanup redirects. Root-level middlewares apply to the custom handler.
|
||||
|
||||
Legacy note: `DisableNotFoundHandler()` is now a no‑op and preserved only for API compatibility.
|
||||
`DisableNotFoundHandler()` has no effect.
|
||||
|
||||
### Middleware Ordering
|
||||
|
||||
@@ -289,7 +289,7 @@ http.ListenAndServe(":8080", mux)
|
||||
- Wrong method on an existing path returns `405 Method Not Allowed` (with an `Allow` header).
|
||||
- Unknown path returns `404 Not Found`.
|
||||
|
||||
You can optionally configure a custom 404 handler with `NotFoundHandler(fn)`. It will run only when no route matches and does not affect 405 handling. The custom handler will have global middlewares applied to it. The legacy `DisableNotFoundHandler()` is now a no‑op and kept only for compatibility.
|
||||
You can optionally configure a custom 404 handler with `NotFoundHandler(fn)`. It runs only when `http.ServeMux` would return 404 and does not affect method mismatch responses or path-cleanup redirects. Global middlewares apply to the custom handler. `DisableNotFoundHandler()` has no effect.
|
||||
|
||||
### HandleFiles helper
|
||||
|
||||
|
||||
+17
-12
@@ -63,14 +63,19 @@ func (b *Bundle) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// but intercept 404s to use custom handler if provided
|
||||
muxHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if pattern == "" && root.notFound != nil {
|
||||
// no route matched, need to check if it's a true 404 or a 405
|
||||
// probe the mux to see what status it would return
|
||||
probe := &statusRecorder{status: http.StatusOK}
|
||||
b.mux.ServeHTTP(probe, r)
|
||||
handler, currentPattern := b.mux.Handler(r)
|
||||
if currentPattern != "" {
|
||||
b.mux.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// if mux wants to return 405 (Method Not Allowed), let it handle the request
|
||||
// to preserve the proper 405 response and Allow header
|
||||
if probe.status == http.StatusMethodNotAllowed {
|
||||
// no route matched, need to check if it's a true 404
|
||||
// probe the synthetic handler to see what status it would return
|
||||
probe := &statusRecorder{status: http.StatusOK}
|
||||
handler.ServeHTTP(probe, r)
|
||||
|
||||
// let the mux handle redirects and method mismatches
|
||||
if probe.status != http.StatusNotFound {
|
||||
b.mux.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
@@ -181,13 +186,13 @@ func (b *Bundle) Handler(r *http.Request) (h http.Handler, pattern string) {
|
||||
return b.mux.Handler(r)
|
||||
}
|
||||
|
||||
// DisableNotFoundHandler used to disable auto-registration of a catch-all 404.
|
||||
// Deprecated: now a no-op retained for API compatibility.
|
||||
// DisableNotFoundHandler has no effect.
|
||||
//
|
||||
// Deprecated: this method has no effect.
|
||||
func (b *Bundle) DisableNotFoundHandler() {}
|
||||
|
||||
// NotFoundHandler sets a custom handler for any unmatched routes (404 responses).
|
||||
// Note: This handler is only used for true 404s. Requests to valid paths with
|
||||
// incorrect HTTP methods will still return 405 Method Not Allowed with Allow header.
|
||||
// NotFoundHandler sets a custom handler for true 404 responses.
|
||||
// Method mismatches and path-cleanup redirects are handled by http.ServeMux.
|
||||
func (b *Bundle) NotFoundHandler(handler http.HandlerFunc) {
|
||||
// always set on the root bundle so custom 404 works regardless of which bundle serves.
|
||||
if b.root != nil {
|
||||
|
||||
+89
-80
@@ -1,83 +1,92 @@
|
||||
linters-settings:
|
||||
govet:
|
||||
check-shadowing: true
|
||||
golint:
|
||||
min-confidence: 0.6
|
||||
gocyclo:
|
||||
min-complexity: 15
|
||||
maligned:
|
||||
suggest-new: true
|
||||
dupl:
|
||||
threshold: 100
|
||||
goconst:
|
||||
min-len: 2
|
||||
min-occurrences: 2
|
||||
misspell:
|
||||
locale: US
|
||||
lll:
|
||||
line-length: 140
|
||||
gocritic:
|
||||
enabled-tags:
|
||||
- performance
|
||||
- style
|
||||
- experimental
|
||||
disabled-checks:
|
||||
- wrapperFunc
|
||||
- hugeParam
|
||||
- rangeValCopy
|
||||
|
||||
linters:
|
||||
disable-all: true
|
||||
enable:
|
||||
- megacheck
|
||||
- revive
|
||||
- govet
|
||||
- unconvert
|
||||
- gas
|
||||
- misspell
|
||||
- unused
|
||||
- typecheck
|
||||
- ineffassign
|
||||
- stylecheck
|
||||
- gochecknoinits
|
||||
- exportloopref
|
||||
- nakedret
|
||||
- gosimple
|
||||
- prealloc
|
||||
|
||||
fast: false
|
||||
|
||||
|
||||
version: "2"
|
||||
run:
|
||||
# modules-download-mode: vendor
|
||||
skip-dirs:
|
||||
- vendor
|
||||
concurrency: 4
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- gochecknoinits
|
||||
- gocritic
|
||||
- gosec
|
||||
- govet
|
||||
- ineffassign
|
||||
- nakedret
|
||||
- prealloc
|
||||
- revive
|
||||
- staticcheck
|
||||
- unconvert
|
||||
- unparam
|
||||
- unused
|
||||
settings:
|
||||
goconst:
|
||||
min-len: 2
|
||||
min-occurrences: 2
|
||||
gocritic:
|
||||
disabled-checks:
|
||||
- wrapperFunc
|
||||
- hugeParam
|
||||
- rangeValCopy
|
||||
enabled-tags:
|
||||
- performance
|
||||
- style
|
||||
- experimental
|
||||
gocyclo:
|
||||
min-complexity: 15
|
||||
govet:
|
||||
enable-all: true
|
||||
disable:
|
||||
- fieldalignment
|
||||
lll:
|
||||
line-length: 140
|
||||
misspell:
|
||||
locale: US
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
- text: "should have a package comment, unless it's in another file for this package"
|
||||
linters:
|
||||
- golint
|
||||
- text: "exitAfterDefer:"
|
||||
linters:
|
||||
- gocritic
|
||||
- text: "whyNoLint: include an explanation for nolint directive"
|
||||
linters:
|
||||
- gocritic
|
||||
- text: "go.mongodb.org/mongo-driver/bson/primitive.E"
|
||||
linters:
|
||||
- govet
|
||||
- text: "weak cryptographic primitive"
|
||||
linters:
|
||||
- gosec
|
||||
- text: "at least one file in a package should have a package comment"
|
||||
linters:
|
||||
- stylecheck
|
||||
- text: "should have a package comment"
|
||||
linters:
|
||||
- revive
|
||||
- text: 'Deferring unsafe method "Close" on type "io.ReadCloser"'
|
||||
linters:
|
||||
- gosec
|
||||
exclude-use-default: false
|
||||
exclusions:
|
||||
generated: lax
|
||||
rules:
|
||||
- linters:
|
||||
- gocritic
|
||||
text: "exitAfterDefer:"
|
||||
- linters:
|
||||
- gocritic
|
||||
text: "whyNoLint: include an explanation for nolint directive"
|
||||
- linters:
|
||||
- govet
|
||||
text: "go.mongodb.org/mongo-driver/bson/primitive.E"
|
||||
- linters:
|
||||
- gosec
|
||||
text: "weak cryptographic primitive"
|
||||
- linters:
|
||||
- gosec
|
||||
text: "integer overflow conversion"
|
||||
- linters:
|
||||
- revive
|
||||
text: "should have a package comment"
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: "at least one file in a package should have a package comment"
|
||||
- linters:
|
||||
- gocritic
|
||||
text: "commentedOutCode: may want to remove commented-out code"
|
||||
- linters:
|
||||
- gocritic
|
||||
text: "unnamedResult: consider giving a name to these results"
|
||||
- linters:
|
||||
- revive
|
||||
text: "var-naming: don't use an underscore in package name"
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: "should not use underscores in package names"
|
||||
- linters:
|
||||
- govet
|
||||
text: "struct literal uses unkeyed fields"
|
||||
- linters:
|
||||
- unparam
|
||||
- unused
|
||||
- revive
|
||||
path: _test\.go$
|
||||
text: "unused-parameter"
|
||||
paths:
|
||||
- vendor
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
|
||||
+9
-1
@@ -72,7 +72,7 @@ It can work as a regular errgrp.Group or with early termination. It is thread-sa
|
||||
```go
|
||||
ewg := syncs.NewErrSizedGroup(5, syncs.Preemptive) // error wait group with max size=5, don't try to start more if any error happened
|
||||
for i :=0; i<10; i++ {
|
||||
ewg.Go(func(ctx context.Context) error { // Go here could be blocked if trying to run >5 at the same time
|
||||
ewg.Go(func() error { // Go here could be blocked if trying to run >5 at the same time
|
||||
err := doThings(ctx) // only 5 of these will run in parallel
|
||||
return err
|
||||
})
|
||||
@@ -80,3 +80,11 @@ It can work as a regular errgrp.Group or with early termination. It is thread-sa
|
||||
err := ewg.Wait()
|
||||
```
|
||||
|
||||
`Wait` returns all the collected errors as `*MultiError`, which implements `Unwrap() []error`, so `errors.Is` and `errors.As` match any of them:
|
||||
|
||||
```go
|
||||
if err := ewg.Wait(); errors.Is(err, context.Canceled) {
|
||||
// at least one of the goroutines was canceled
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
+12
-9
@@ -2,6 +2,7 @@ package syncs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
@@ -15,7 +16,6 @@ type ErrSizedGroup struct {
|
||||
sema Locker
|
||||
|
||||
err *MultiError
|
||||
errLock sync.RWMutex
|
||||
errOnce sync.Once
|
||||
}
|
||||
|
||||
@@ -88,8 +88,6 @@ func (g *ErrSizedGroup) Go(f func() error) {
|
||||
if !g.termOnError {
|
||||
return false
|
||||
}
|
||||
g.errLock.RLock()
|
||||
defer g.errLock.RUnlock()
|
||||
return g.err.ErrorOrNil() != nil
|
||||
}
|
||||
|
||||
@@ -109,9 +107,7 @@ func (g *ErrSizedGroup) Go(f func() error) {
|
||||
}
|
||||
|
||||
if err := f(); err != nil {
|
||||
g.errLock.Lock()
|
||||
g.err = g.err.append(err)
|
||||
g.errLock.Unlock()
|
||||
g.err.append(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -129,11 +125,10 @@ type MultiError struct {
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
func (m *MultiError) append(err error) *MultiError {
|
||||
func (m *MultiError) append(err error) {
|
||||
m.lock.Lock()
|
||||
m.errors = append(m.errors, err)
|
||||
m.lock.Unlock()
|
||||
return m
|
||||
}
|
||||
|
||||
// ErrorOrNil returns nil if no errors or multierror if errors occurred
|
||||
@@ -162,8 +157,16 @@ func (m *MultiError) Error() string {
|
||||
return fmt.Sprintf("%d error(s) occurred: %s", len(m.errors), strings.Join(errs, ", "))
|
||||
}
|
||||
|
||||
// Errors returns all errors collected
|
||||
func (m *MultiError) Errors() []error {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
return m.errors
|
||||
return slices.Clone(m.errors)
|
||||
}
|
||||
|
||||
// Unwrap returns all errors collected, allows errors.Is and errors.As to match any of them
|
||||
func (m *MultiError) Unwrap() []error {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
return slices.Clone(m.errors)
|
||||
}
|
||||
|
||||
+1
-2
@@ -8,9 +8,8 @@ type Locker interface {
|
||||
TryLock() bool
|
||||
}
|
||||
|
||||
// Semaphore implementation, counted lock only. Implements sync.Locker interface, thread safe.
|
||||
// Semaphore implementation, counted lock only. Implements Locker interface, thread safe.
|
||||
type semaphore struct {
|
||||
Locker
|
||||
ch chan struct{}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ type SizedGroup struct {
|
||||
// NewSizedGroup makes wait group with limited size alive goroutines
|
||||
func NewSizedGroup(size int, opts ...GroupOption) *SizedGroup {
|
||||
res := SizedGroup{sema: NewSemaphore(size)}
|
||||
res.options.ctx = context.Background()
|
||||
res.ctx = context.Background()
|
||||
for _, opt := range opts {
|
||||
opt(&res.options)
|
||||
}
|
||||
|
||||
-354
@@ -1,354 +0,0 @@
|
||||
Mozilla Public License, version 2.0
|
||||
|
||||
1. Definitions
|
||||
|
||||
1.1. “Contributor”
|
||||
|
||||
means each individual or legal entity that creates, contributes to the
|
||||
creation of, or owns Covered Software.
|
||||
|
||||
1.2. “Contributor Version”
|
||||
|
||||
means the combination of the Contributions of others (if any) used by a
|
||||
Contributor and that particular Contributor’s Contribution.
|
||||
|
||||
1.3. “Contribution”
|
||||
|
||||
means Covered Software of a particular Contributor.
|
||||
|
||||
1.4. “Covered Software”
|
||||
|
||||
means Source Code Form to which the initial Contributor has attached the
|
||||
notice in Exhibit A, the Executable Form of such Source Code Form, and
|
||||
Modifications of such Source Code Form, in each case including portions
|
||||
thereof.
|
||||
|
||||
1.5. “Incompatible With Secondary Licenses”
|
||||
means
|
||||
|
||||
a. that the initial Contributor has attached the notice described in
|
||||
Exhibit B to the Covered Software; or
|
||||
|
||||
b. that the Covered Software was made available under the terms of version
|
||||
1.1 or earlier of the License, but not also under the terms of a
|
||||
Secondary License.
|
||||
|
||||
1.6. “Executable Form”
|
||||
|
||||
means any form of the work other than Source Code Form.
|
||||
|
||||
1.7. “Larger Work”
|
||||
|
||||
means a work that combines Covered Software with other material, in a separate
|
||||
file or files, that is not Covered Software.
|
||||
|
||||
1.8. “License”
|
||||
|
||||
means this document.
|
||||
|
||||
1.9. “Licensable”
|
||||
|
||||
means having the right to grant, to the maximum extent possible, whether at the
|
||||
time of the initial grant or subsequently, any and all of the rights conveyed by
|
||||
this License.
|
||||
|
||||
1.10. “Modifications”
|
||||
|
||||
means any of the following:
|
||||
|
||||
a. any file in Source Code Form that results from an addition to, deletion
|
||||
from, or modification of the contents of Covered Software; or
|
||||
|
||||
b. any new file in Source Code Form that contains any Covered Software.
|
||||
|
||||
1.11. “Patent Claims” of a Contributor
|
||||
|
||||
means any patent claim(s), including without limitation, method, process,
|
||||
and apparatus claims, in any patent Licensable by such Contributor that
|
||||
would be infringed, but for the grant of the License, by the making,
|
||||
using, selling, offering for sale, having made, import, or transfer of
|
||||
either its Contributions or its Contributor Version.
|
||||
|
||||
1.12. “Secondary License”
|
||||
|
||||
means either the GNU General Public License, Version 2.0, the GNU Lesser
|
||||
General Public License, Version 2.1, the GNU Affero General Public
|
||||
License, Version 3.0, or any later versions of those licenses.
|
||||
|
||||
1.13. “Source Code Form”
|
||||
|
||||
means the form of the work preferred for making modifications.
|
||||
|
||||
1.14. “You” (or “Your”)
|
||||
|
||||
means an individual or a legal entity exercising rights under this
|
||||
License. For legal entities, “You” includes any entity that controls, is
|
||||
controlled by, or is under common control with You. For purposes of this
|
||||
definition, “control” means (a) the power, direct or indirect, to cause
|
||||
the direction or management of such entity, whether by contract or
|
||||
otherwise, or (b) ownership of more than fifty percent (50%) of the
|
||||
outstanding shares or beneficial ownership of such entity.
|
||||
|
||||
|
||||
2. License Grants and Conditions
|
||||
|
||||
2.1. Grants
|
||||
|
||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
||||
non-exclusive license:
|
||||
|
||||
a. under intellectual property rights (other than patent or trademark)
|
||||
Licensable by such Contributor to use, reproduce, make available,
|
||||
modify, display, perform, distribute, and otherwise exploit its
|
||||
Contributions, either on an unmodified basis, with Modifications, or as
|
||||
part of a Larger Work; and
|
||||
|
||||
b. under Patent Claims of such Contributor to make, use, sell, offer for
|
||||
sale, have made, import, and otherwise transfer either its Contributions
|
||||
or its Contributor Version.
|
||||
|
||||
2.2. Effective Date
|
||||
|
||||
The licenses granted in Section 2.1 with respect to any Contribution become
|
||||
effective for each Contribution on the date the Contributor first distributes
|
||||
such Contribution.
|
||||
|
||||
2.3. Limitations on Grant Scope
|
||||
|
||||
The licenses granted in this Section 2 are the only rights granted under this
|
||||
License. No additional rights or licenses will be implied from the distribution
|
||||
or licensing of Covered Software under this License. Notwithstanding Section
|
||||
2.1(b) above, no patent license is granted by a Contributor:
|
||||
|
||||
a. for any code that a Contributor has removed from Covered Software; or
|
||||
|
||||
b. for infringements caused by: (i) Your and any other third party’s
|
||||
modifications of Covered Software, or (ii) the combination of its
|
||||
Contributions with other software (except as part of its Contributor
|
||||
Version); or
|
||||
|
||||
c. under Patent Claims infringed by Covered Software in the absence of its
|
||||
Contributions.
|
||||
|
||||
This License does not grant any rights in the trademarks, service marks, or
|
||||
logos of any Contributor (except as may be necessary to comply with the
|
||||
notice requirements in Section 3.4).
|
||||
|
||||
2.4. Subsequent Licenses
|
||||
|
||||
No Contributor makes additional grants as a result of Your choice to
|
||||
distribute the Covered Software under a subsequent version of this License
|
||||
(see Section 10.2) or under the terms of a Secondary License (if permitted
|
||||
under the terms of Section 3.3).
|
||||
|
||||
2.5. Representation
|
||||
|
||||
Each Contributor represents that the Contributor believes its Contributions
|
||||
are its original creation(s) or it has sufficient rights to grant the
|
||||
rights to its Contributions conveyed by this License.
|
||||
|
||||
2.6. Fair Use
|
||||
|
||||
This License is not intended to limit any rights You have under applicable
|
||||
copyright doctrines of fair use, fair dealing, or other equivalents.
|
||||
|
||||
2.7. Conditions
|
||||
|
||||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
|
||||
Section 2.1.
|
||||
|
||||
|
||||
3. Responsibilities
|
||||
|
||||
3.1. Distribution of Source Form
|
||||
|
||||
All distribution of Covered Software in Source Code Form, including any
|
||||
Modifications that You create or to which You contribute, must be under the
|
||||
terms of this License. You must inform recipients that the Source Code Form
|
||||
of the Covered Software is governed by the terms of this License, and how
|
||||
they can obtain a copy of this License. You may not attempt to alter or
|
||||
restrict the recipients’ rights in the Source Code Form.
|
||||
|
||||
3.2. Distribution of Executable Form
|
||||
|
||||
If You distribute Covered Software in Executable Form then:
|
||||
|
||||
a. such Covered Software must also be made available in Source Code Form,
|
||||
as described in Section 3.1, and You must inform recipients of the
|
||||
Executable Form how they can obtain a copy of such Source Code Form by
|
||||
reasonable means in a timely manner, at a charge no more than the cost
|
||||
of distribution to the recipient; and
|
||||
|
||||
b. You may distribute such Executable Form under the terms of this License,
|
||||
or sublicense it under different terms, provided that the license for
|
||||
the Executable Form does not attempt to limit or alter the recipients’
|
||||
rights in the Source Code Form under this License.
|
||||
|
||||
3.3. Distribution of a Larger Work
|
||||
|
||||
You may create and distribute a Larger Work under terms of Your choice,
|
||||
provided that You also comply with the requirements of this License for the
|
||||
Covered Software. If the Larger Work is a combination of Covered Software
|
||||
with a work governed by one or more Secondary Licenses, and the Covered
|
||||
Software is not Incompatible With Secondary Licenses, this License permits
|
||||
You to additionally distribute such Covered Software under the terms of
|
||||
such Secondary License(s), so that the recipient of the Larger Work may, at
|
||||
their option, further distribute the Covered Software under the terms of
|
||||
either this License or such Secondary License(s).
|
||||
|
||||
3.4. Notices
|
||||
|
||||
You may not remove or alter the substance of any license notices (including
|
||||
copyright notices, patent notices, disclaimers of warranty, or limitations
|
||||
of liability) contained within the Source Code Form of the Covered
|
||||
Software, except that You may alter any license notices to the extent
|
||||
required to remedy known factual inaccuracies.
|
||||
|
||||
3.5. Application of Additional Terms
|
||||
|
||||
You may choose to offer, and to charge a fee for, warranty, support,
|
||||
indemnity or liability obligations to one or more recipients of Covered
|
||||
Software. However, You may do so only on Your own behalf, and not on behalf
|
||||
of any Contributor. You must make it absolutely clear that any such
|
||||
warranty, support, indemnity, or liability obligation is offered by You
|
||||
alone, and You hereby agree to indemnify every Contributor for any
|
||||
liability incurred by such Contributor as a result of warranty, support,
|
||||
indemnity or liability terms You offer. You may include additional
|
||||
disclaimers of warranty and limitations of liability specific to any
|
||||
jurisdiction.
|
||||
|
||||
4. Inability to Comply Due to Statute or Regulation
|
||||
|
||||
If it is impossible for You to comply with any of the terms of this License
|
||||
with respect to some or all of the Covered Software due to statute, judicial
|
||||
order, or regulation then You must: (a) comply with the terms of this License
|
||||
to the maximum extent possible; and (b) describe the limitations and the code
|
||||
they affect. Such description must be placed in a text file included with all
|
||||
distributions of the Covered Software under this License. Except to the
|
||||
extent prohibited by statute or regulation, such description must be
|
||||
sufficiently detailed for a recipient of ordinary skill to be able to
|
||||
understand it.
|
||||
|
||||
5. Termination
|
||||
|
||||
5.1. The rights granted under this License will terminate automatically if You
|
||||
fail to comply with any of its terms. However, if You become compliant,
|
||||
then the rights granted under this License from a particular Contributor
|
||||
are reinstated (a) provisionally, unless and until such Contributor
|
||||
explicitly and finally terminates Your grants, and (b) on an ongoing basis,
|
||||
if such Contributor fails to notify You of the non-compliance by some
|
||||
reasonable means prior to 60 days after You have come back into compliance.
|
||||
Moreover, Your grants from a particular Contributor are reinstated on an
|
||||
ongoing basis if such Contributor notifies You of the non-compliance by
|
||||
some reasonable means, this is the first time You have received notice of
|
||||
non-compliance with this License from such Contributor, and You become
|
||||
compliant prior to 30 days after Your receipt of the notice.
|
||||
|
||||
5.2. If You initiate litigation against any entity by asserting a patent
|
||||
infringement claim (excluding declaratory judgment actions, counter-claims,
|
||||
and cross-claims) alleging that a Contributor Version directly or
|
||||
indirectly infringes any patent, then the rights granted to You by any and
|
||||
all Contributors for the Covered Software under Section 2.1 of this License
|
||||
shall terminate.
|
||||
|
||||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
|
||||
license agreements (excluding distributors and resellers) which have been
|
||||
validly granted by You or Your distributors under this License prior to
|
||||
termination shall survive termination.
|
||||
|
||||
6. Disclaimer of Warranty
|
||||
|
||||
Covered Software is provided under this License on an “as is” basis, without
|
||||
warranty of any kind, either expressed, implied, or statutory, including,
|
||||
without limitation, warranties that the Covered Software is free of defects,
|
||||
merchantable, fit for a particular purpose or non-infringing. The entire
|
||||
risk as to the quality and performance of the Covered Software is with You.
|
||||
Should any Covered Software prove defective in any respect, You (not any
|
||||
Contributor) assume the cost of any necessary servicing, repair, or
|
||||
correction. This disclaimer of warranty constitutes an essential part of this
|
||||
License. No use of any Covered Software is authorized under this License
|
||||
except under this disclaimer.
|
||||
|
||||
7. Limitation of Liability
|
||||
|
||||
Under no circumstances and under no legal theory, whether tort (including
|
||||
negligence), contract, or otherwise, shall any Contributor, or anyone who
|
||||
distributes Covered Software as permitted above, be liable to You for any
|
||||
direct, indirect, special, incidental, or consequential damages of any
|
||||
character including, without limitation, damages for lost profits, loss of
|
||||
goodwill, work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses, even if such party shall have been
|
||||
informed of the possibility of such damages. This limitation of liability
|
||||
shall not apply to liability for death or personal injury resulting from such
|
||||
party’s negligence to the extent applicable law prohibits such limitation.
|
||||
Some jurisdictions do not allow the exclusion or limitation of incidental or
|
||||
consequential damages, so this exclusion and limitation may not apply to You.
|
||||
|
||||
8. Litigation
|
||||
|
||||
Any litigation relating to this License may be brought only in the courts of
|
||||
a jurisdiction where the defendant maintains its principal place of business
|
||||
and such litigation shall be governed by laws of that jurisdiction, without
|
||||
reference to its conflict-of-law provisions. Nothing in this Section shall
|
||||
prevent a party’s ability to bring cross-claims or counter-claims.
|
||||
|
||||
9. Miscellaneous
|
||||
|
||||
This License represents the complete agreement concerning the subject matter
|
||||
hereof. If any provision of this License is held to be unenforceable, such
|
||||
provision shall be reformed only to the extent necessary to make it
|
||||
enforceable. Any law or regulation which provides that the language of a
|
||||
contract shall be construed against the drafter shall not be used to construe
|
||||
this License against a Contributor.
|
||||
|
||||
|
||||
10. Versions of the License
|
||||
|
||||
10.1. New Versions
|
||||
|
||||
Mozilla Foundation is the license steward. Except as provided in Section
|
||||
10.3, no one other than the license steward has the right to modify or
|
||||
publish new versions of this License. Each version will be given a
|
||||
distinguishing version number.
|
||||
|
||||
10.2. Effect of New Versions
|
||||
|
||||
You may distribute the Covered Software under the terms of the version of
|
||||
the License under which You originally received the Covered Software, or
|
||||
under the terms of any subsequent version published by the license
|
||||
steward.
|
||||
|
||||
10.3. Modified Versions
|
||||
|
||||
If you create software not governed by this License, and you want to
|
||||
create a new license for such software, you may create and use a modified
|
||||
version of this License if you rename the license and remove any
|
||||
references to the name of the license steward (except to note that such
|
||||
modified license differs from this License).
|
||||
|
||||
10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses
|
||||
If You choose to distribute Source Code Form that is Incompatible With
|
||||
Secondary Licenses under the terms of this version of the License, the
|
||||
notice described in Exhibit B of this License must be attached.
|
||||
|
||||
Exhibit A - Source Code Form License Notice
|
||||
|
||||
This Source Code Form is subject to the
|
||||
terms of the Mozilla Public License, v.
|
||||
2.0. If a copy of the MPL was not
|
||||
distributed with this file, You can
|
||||
obtain one at
|
||||
http://mozilla.org/MPL/2.0/.
|
||||
|
||||
If it is not possible or desirable to put the notice in a particular file, then
|
||||
You may include the notice in a location (such as a LICENSE file in a relevant
|
||||
directory) where a recipient would be likely to look for such a notice.
|
||||
|
||||
You may add additional accurate notices of copyright ownership.
|
||||
|
||||
Exhibit B - “Incompatible With Secondary Licenses” Notice
|
||||
|
||||
This Source Code Form is “Incompatible
|
||||
With Secondary Licenses”, as defined by
|
||||
the Mozilla Public License, v. 2.0.
|
||||
|
||||
-89
@@ -1,89 +0,0 @@
|
||||
# errwrap
|
||||
|
||||
`errwrap` is a package for Go that formalizes the pattern of wrapping errors
|
||||
and checking if an error contains another error.
|
||||
|
||||
There is a common pattern in Go of taking a returned `error` value and
|
||||
then wrapping it (such as with `fmt.Errorf`) before returning it. The problem
|
||||
with this pattern is that you completely lose the original `error` structure.
|
||||
|
||||
Arguably the _correct_ approach is that you should make a custom structure
|
||||
implementing the `error` interface, and have the original error as a field
|
||||
on that structure, such [as this example](http://golang.org/pkg/os/#PathError).
|
||||
This is a good approach, but you have to know the entire chain of possible
|
||||
rewrapping that happens, when you might just care about one.
|
||||
|
||||
`errwrap` formalizes this pattern (it doesn't matter what approach you use
|
||||
above) by giving a single interface for wrapping errors, checking if a specific
|
||||
error is wrapped, and extracting that error.
|
||||
|
||||
## Installation and Docs
|
||||
|
||||
Install using `go get github.com/hashicorp/errwrap`.
|
||||
|
||||
Full documentation is available at
|
||||
http://godoc.org/github.com/hashicorp/errwrap
|
||||
|
||||
## Usage
|
||||
|
||||
#### Basic Usage
|
||||
|
||||
Below is a very basic example of its usage:
|
||||
|
||||
```go
|
||||
// A function that always returns an error, but wraps it, like a real
|
||||
// function might.
|
||||
func tryOpen() error {
|
||||
_, err := os.Open("/i/dont/exist")
|
||||
if err != nil {
|
||||
return errwrap.Wrapf("Doesn't exist: {{err}}", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
err := tryOpen()
|
||||
|
||||
// We can use the Contains helpers to check if an error contains
|
||||
// another error. It is safe to do this with a nil error, or with
|
||||
// an error that doesn't even use the errwrap package.
|
||||
if errwrap.Contains(err, "does not exist") {
|
||||
// Do something
|
||||
}
|
||||
if errwrap.ContainsType(err, new(os.PathError)) {
|
||||
// Do something
|
||||
}
|
||||
|
||||
// Or we can use the associated `Get` functions to just extract
|
||||
// a specific error. This would return nil if that specific error doesn't
|
||||
// exist.
|
||||
perr := errwrap.GetType(err, new(os.PathError))
|
||||
}
|
||||
```
|
||||
|
||||
#### Custom Types
|
||||
|
||||
If you're already making custom types that properly wrap errors, then
|
||||
you can get all the functionality of `errwraps.Contains` and such by
|
||||
implementing the `Wrapper` interface with just one function. Example:
|
||||
|
||||
```go
|
||||
type AppError {
|
||||
Code ErrorCode
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *AppError) WrappedErrors() []error {
|
||||
return []error{e.Err}
|
||||
}
|
||||
```
|
||||
|
||||
Now this works:
|
||||
|
||||
```go
|
||||
err := &AppError{Err: fmt.Errorf("an error")}
|
||||
if errwrap.ContainsType(err, fmt.Errorf("")) {
|
||||
// This will work!
|
||||
}
|
||||
```
|
||||
-178
@@ -1,178 +0,0 @@
|
||||
// Package errwrap implements methods to formalize error wrapping in Go.
|
||||
//
|
||||
// All of the top-level functions that take an `error` are built to be able
|
||||
// to take any error, not just wrapped errors. This allows you to use errwrap
|
||||
// without having to type-check and type-cast everywhere.
|
||||
package errwrap
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// WalkFunc is the callback called for Walk.
|
||||
type WalkFunc func(error)
|
||||
|
||||
// Wrapper is an interface that can be implemented by custom types to
|
||||
// have all the Contains, Get, etc. functions in errwrap work.
|
||||
//
|
||||
// When Walk reaches a Wrapper, it will call the callback for every
|
||||
// wrapped error in addition to the wrapper itself. Since all the top-level
|
||||
// functions in errwrap use Walk, this means that all those functions work
|
||||
// with your custom type.
|
||||
type Wrapper interface {
|
||||
WrappedErrors() []error
|
||||
}
|
||||
|
||||
// Wrap defines that outer wraps inner, returning an error type that
|
||||
// can be cleanly used with the other methods in this package, such as
|
||||
// Contains, GetAll, etc.
|
||||
//
|
||||
// This function won't modify the error message at all (the outer message
|
||||
// will be used).
|
||||
func Wrap(outer, inner error) error {
|
||||
return &wrappedError{
|
||||
Outer: outer,
|
||||
Inner: inner,
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapf wraps an error with a formatting message. This is similar to using
|
||||
// `fmt.Errorf` to wrap an error. If you're using `fmt.Errorf` to wrap
|
||||
// errors, you should replace it with this.
|
||||
//
|
||||
// format is the format of the error message. The string '{{err}}' will
|
||||
// be replaced with the original error message.
|
||||
//
|
||||
// Deprecated: Use fmt.Errorf()
|
||||
func Wrapf(format string, err error) error {
|
||||
outerMsg := "<nil>"
|
||||
if err != nil {
|
||||
outerMsg = err.Error()
|
||||
}
|
||||
|
||||
outer := errors.New(strings.Replace(
|
||||
format, "{{err}}", outerMsg, -1))
|
||||
|
||||
return Wrap(outer, err)
|
||||
}
|
||||
|
||||
// Contains checks if the given error contains an error with the
|
||||
// message msg. If err is not a wrapped error, this will always return
|
||||
// false unless the error itself happens to match this msg.
|
||||
func Contains(err error, msg string) bool {
|
||||
return len(GetAll(err, msg)) > 0
|
||||
}
|
||||
|
||||
// ContainsType checks if the given error contains an error with
|
||||
// the same concrete type as v. If err is not a wrapped error, this will
|
||||
// check the err itself.
|
||||
func ContainsType(err error, v interface{}) bool {
|
||||
return len(GetAllType(err, v)) > 0
|
||||
}
|
||||
|
||||
// Get is the same as GetAll but returns the deepest matching error.
|
||||
func Get(err error, msg string) error {
|
||||
es := GetAll(err, msg)
|
||||
if len(es) > 0 {
|
||||
return es[len(es)-1]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetType is the same as GetAllType but returns the deepest matching error.
|
||||
func GetType(err error, v interface{}) error {
|
||||
es := GetAllType(err, v)
|
||||
if len(es) > 0 {
|
||||
return es[len(es)-1]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAll gets all the errors that might be wrapped in err with the
|
||||
// given message. The order of the errors is such that the outermost
|
||||
// matching error (the most recent wrap) is index zero, and so on.
|
||||
func GetAll(err error, msg string) []error {
|
||||
var result []error
|
||||
|
||||
Walk(err, func(err error) {
|
||||
if err.Error() == msg {
|
||||
result = append(result, err)
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// GetAllType gets all the errors that are the same type as v.
|
||||
//
|
||||
// The order of the return value is the same as described in GetAll.
|
||||
func GetAllType(err error, v interface{}) []error {
|
||||
var result []error
|
||||
|
||||
var search string
|
||||
if v != nil {
|
||||
search = reflect.TypeOf(v).String()
|
||||
}
|
||||
Walk(err, func(err error) {
|
||||
var needle string
|
||||
if err != nil {
|
||||
needle = reflect.TypeOf(err).String()
|
||||
}
|
||||
|
||||
if needle == search {
|
||||
result = append(result, err)
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Walk walks all the wrapped errors in err and calls the callback. If
|
||||
// err isn't a wrapped error, this will be called once for err. If err
|
||||
// is a wrapped error, the callback will be called for both the wrapper
|
||||
// that implements error as well as the wrapped error itself.
|
||||
func Walk(err error, cb WalkFunc) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch e := err.(type) {
|
||||
case *wrappedError:
|
||||
cb(e.Outer)
|
||||
Walk(e.Inner, cb)
|
||||
case Wrapper:
|
||||
cb(err)
|
||||
|
||||
for _, err := range e.WrappedErrors() {
|
||||
Walk(err, cb)
|
||||
}
|
||||
case interface{ Unwrap() error }:
|
||||
cb(err)
|
||||
Walk(e.Unwrap(), cb)
|
||||
default:
|
||||
cb(err)
|
||||
}
|
||||
}
|
||||
|
||||
// wrappedError is an implementation of error that has both the
|
||||
// outer and inner errors.
|
||||
type wrappedError struct {
|
||||
Outer error
|
||||
Inner error
|
||||
}
|
||||
|
||||
func (w *wrappedError) Error() string {
|
||||
return w.Outer.Error()
|
||||
}
|
||||
|
||||
func (w *wrappedError) WrappedErrors() []error {
|
||||
return []error{w.Outer, w.Inner}
|
||||
}
|
||||
|
||||
func (w *wrappedError) Unwrap() error {
|
||||
return w.Inner
|
||||
}
|
||||
-353
@@ -1,353 +0,0 @@
|
||||
Mozilla Public License, version 2.0
|
||||
|
||||
1. Definitions
|
||||
|
||||
1.1. “Contributor”
|
||||
|
||||
means each individual or legal entity that creates, contributes to the
|
||||
creation of, or owns Covered Software.
|
||||
|
||||
1.2. “Contributor Version”
|
||||
|
||||
means the combination of the Contributions of others (if any) used by a
|
||||
Contributor and that particular Contributor’s Contribution.
|
||||
|
||||
1.3. “Contribution”
|
||||
|
||||
means Covered Software of a particular Contributor.
|
||||
|
||||
1.4. “Covered Software”
|
||||
|
||||
means Source Code Form to which the initial Contributor has attached the
|
||||
notice in Exhibit A, the Executable Form of such Source Code Form, and
|
||||
Modifications of such Source Code Form, in each case including portions
|
||||
thereof.
|
||||
|
||||
1.5. “Incompatible With Secondary Licenses”
|
||||
means
|
||||
|
||||
a. that the initial Contributor has attached the notice described in
|
||||
Exhibit B to the Covered Software; or
|
||||
|
||||
b. that the Covered Software was made available under the terms of version
|
||||
1.1 or earlier of the License, but not also under the terms of a
|
||||
Secondary License.
|
||||
|
||||
1.6. “Executable Form”
|
||||
|
||||
means any form of the work other than Source Code Form.
|
||||
|
||||
1.7. “Larger Work”
|
||||
|
||||
means a work that combines Covered Software with other material, in a separate
|
||||
file or files, that is not Covered Software.
|
||||
|
||||
1.8. “License”
|
||||
|
||||
means this document.
|
||||
|
||||
1.9. “Licensable”
|
||||
|
||||
means having the right to grant, to the maximum extent possible, whether at the
|
||||
time of the initial grant or subsequently, any and all of the rights conveyed by
|
||||
this License.
|
||||
|
||||
1.10. “Modifications”
|
||||
|
||||
means any of the following:
|
||||
|
||||
a. any file in Source Code Form that results from an addition to, deletion
|
||||
from, or modification of the contents of Covered Software; or
|
||||
|
||||
b. any new file in Source Code Form that contains any Covered Software.
|
||||
|
||||
1.11. “Patent Claims” of a Contributor
|
||||
|
||||
means any patent claim(s), including without limitation, method, process,
|
||||
and apparatus claims, in any patent Licensable by such Contributor that
|
||||
would be infringed, but for the grant of the License, by the making,
|
||||
using, selling, offering for sale, having made, import, or transfer of
|
||||
either its Contributions or its Contributor Version.
|
||||
|
||||
1.12. “Secondary License”
|
||||
|
||||
means either the GNU General Public License, Version 2.0, the GNU Lesser
|
||||
General Public License, Version 2.1, the GNU Affero General Public
|
||||
License, Version 3.0, or any later versions of those licenses.
|
||||
|
||||
1.13. “Source Code Form”
|
||||
|
||||
means the form of the work preferred for making modifications.
|
||||
|
||||
1.14. “You” (or “Your”)
|
||||
|
||||
means an individual or a legal entity exercising rights under this
|
||||
License. For legal entities, “You” includes any entity that controls, is
|
||||
controlled by, or is under common control with You. For purposes of this
|
||||
definition, “control” means (a) the power, direct or indirect, to cause
|
||||
the direction or management of such entity, whether by contract or
|
||||
otherwise, or (b) ownership of more than fifty percent (50%) of the
|
||||
outstanding shares or beneficial ownership of such entity.
|
||||
|
||||
|
||||
2. License Grants and Conditions
|
||||
|
||||
2.1. Grants
|
||||
|
||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
||||
non-exclusive license:
|
||||
|
||||
a. under intellectual property rights (other than patent or trademark)
|
||||
Licensable by such Contributor to use, reproduce, make available,
|
||||
modify, display, perform, distribute, and otherwise exploit its
|
||||
Contributions, either on an unmodified basis, with Modifications, or as
|
||||
part of a Larger Work; and
|
||||
|
||||
b. under Patent Claims of such Contributor to make, use, sell, offer for
|
||||
sale, have made, import, and otherwise transfer either its Contributions
|
||||
or its Contributor Version.
|
||||
|
||||
2.2. Effective Date
|
||||
|
||||
The licenses granted in Section 2.1 with respect to any Contribution become
|
||||
effective for each Contribution on the date the Contributor first distributes
|
||||
such Contribution.
|
||||
|
||||
2.3. Limitations on Grant Scope
|
||||
|
||||
The licenses granted in this Section 2 are the only rights granted under this
|
||||
License. No additional rights or licenses will be implied from the distribution
|
||||
or licensing of Covered Software under this License. Notwithstanding Section
|
||||
2.1(b) above, no patent license is granted by a Contributor:
|
||||
|
||||
a. for any code that a Contributor has removed from Covered Software; or
|
||||
|
||||
b. for infringements caused by: (i) Your and any other third party’s
|
||||
modifications of Covered Software, or (ii) the combination of its
|
||||
Contributions with other software (except as part of its Contributor
|
||||
Version); or
|
||||
|
||||
c. under Patent Claims infringed by Covered Software in the absence of its
|
||||
Contributions.
|
||||
|
||||
This License does not grant any rights in the trademarks, service marks, or
|
||||
logos of any Contributor (except as may be necessary to comply with the
|
||||
notice requirements in Section 3.4).
|
||||
|
||||
2.4. Subsequent Licenses
|
||||
|
||||
No Contributor makes additional grants as a result of Your choice to
|
||||
distribute the Covered Software under a subsequent version of this License
|
||||
(see Section 10.2) or under the terms of a Secondary License (if permitted
|
||||
under the terms of Section 3.3).
|
||||
|
||||
2.5. Representation
|
||||
|
||||
Each Contributor represents that the Contributor believes its Contributions
|
||||
are its original creation(s) or it has sufficient rights to grant the
|
||||
rights to its Contributions conveyed by this License.
|
||||
|
||||
2.6. Fair Use
|
||||
|
||||
This License is not intended to limit any rights You have under applicable
|
||||
copyright doctrines of fair use, fair dealing, or other equivalents.
|
||||
|
||||
2.7. Conditions
|
||||
|
||||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
|
||||
Section 2.1.
|
||||
|
||||
|
||||
3. Responsibilities
|
||||
|
||||
3.1. Distribution of Source Form
|
||||
|
||||
All distribution of Covered Software in Source Code Form, including any
|
||||
Modifications that You create or to which You contribute, must be under the
|
||||
terms of this License. You must inform recipients that the Source Code Form
|
||||
of the Covered Software is governed by the terms of this License, and how
|
||||
they can obtain a copy of this License. You may not attempt to alter or
|
||||
restrict the recipients’ rights in the Source Code Form.
|
||||
|
||||
3.2. Distribution of Executable Form
|
||||
|
||||
If You distribute Covered Software in Executable Form then:
|
||||
|
||||
a. such Covered Software must also be made available in Source Code Form,
|
||||
as described in Section 3.1, and You must inform recipients of the
|
||||
Executable Form how they can obtain a copy of such Source Code Form by
|
||||
reasonable means in a timely manner, at a charge no more than the cost
|
||||
of distribution to the recipient; and
|
||||
|
||||
b. You may distribute such Executable Form under the terms of this License,
|
||||
or sublicense it under different terms, provided that the license for
|
||||
the Executable Form does not attempt to limit or alter the recipients’
|
||||
rights in the Source Code Form under this License.
|
||||
|
||||
3.3. Distribution of a Larger Work
|
||||
|
||||
You may create and distribute a Larger Work under terms of Your choice,
|
||||
provided that You also comply with the requirements of this License for the
|
||||
Covered Software. If the Larger Work is a combination of Covered Software
|
||||
with a work governed by one or more Secondary Licenses, and the Covered
|
||||
Software is not Incompatible With Secondary Licenses, this License permits
|
||||
You to additionally distribute such Covered Software under the terms of
|
||||
such Secondary License(s), so that the recipient of the Larger Work may, at
|
||||
their option, further distribute the Covered Software under the terms of
|
||||
either this License or such Secondary License(s).
|
||||
|
||||
3.4. Notices
|
||||
|
||||
You may not remove or alter the substance of any license notices (including
|
||||
copyright notices, patent notices, disclaimers of warranty, or limitations
|
||||
of liability) contained within the Source Code Form of the Covered
|
||||
Software, except that You may alter any license notices to the extent
|
||||
required to remedy known factual inaccuracies.
|
||||
|
||||
3.5. Application of Additional Terms
|
||||
|
||||
You may choose to offer, and to charge a fee for, warranty, support,
|
||||
indemnity or liability obligations to one or more recipients of Covered
|
||||
Software. However, You may do so only on Your own behalf, and not on behalf
|
||||
of any Contributor. You must make it absolutely clear that any such
|
||||
warranty, support, indemnity, or liability obligation is offered by You
|
||||
alone, and You hereby agree to indemnify every Contributor for any
|
||||
liability incurred by such Contributor as a result of warranty, support,
|
||||
indemnity or liability terms You offer. You may include additional
|
||||
disclaimers of warranty and limitations of liability specific to any
|
||||
jurisdiction.
|
||||
|
||||
4. Inability to Comply Due to Statute or Regulation
|
||||
|
||||
If it is impossible for You to comply with any of the terms of this License
|
||||
with respect to some or all of the Covered Software due to statute, judicial
|
||||
order, or regulation then You must: (a) comply with the terms of this License
|
||||
to the maximum extent possible; and (b) describe the limitations and the code
|
||||
they affect. Such description must be placed in a text file included with all
|
||||
distributions of the Covered Software under this License. Except to the
|
||||
extent prohibited by statute or regulation, such description must be
|
||||
sufficiently detailed for a recipient of ordinary skill to be able to
|
||||
understand it.
|
||||
|
||||
5. Termination
|
||||
|
||||
5.1. The rights granted under this License will terminate automatically if You
|
||||
fail to comply with any of its terms. However, if You become compliant,
|
||||
then the rights granted under this License from a particular Contributor
|
||||
are reinstated (a) provisionally, unless and until such Contributor
|
||||
explicitly and finally terminates Your grants, and (b) on an ongoing basis,
|
||||
if such Contributor fails to notify You of the non-compliance by some
|
||||
reasonable means prior to 60 days after You have come back into compliance.
|
||||
Moreover, Your grants from a particular Contributor are reinstated on an
|
||||
ongoing basis if such Contributor notifies You of the non-compliance by
|
||||
some reasonable means, this is the first time You have received notice of
|
||||
non-compliance with this License from such Contributor, and You become
|
||||
compliant prior to 30 days after Your receipt of the notice.
|
||||
|
||||
5.2. If You initiate litigation against any entity by asserting a patent
|
||||
infringement claim (excluding declaratory judgment actions, counter-claims,
|
||||
and cross-claims) alleging that a Contributor Version directly or
|
||||
indirectly infringes any patent, then the rights granted to You by any and
|
||||
all Contributors for the Covered Software under Section 2.1 of this License
|
||||
shall terminate.
|
||||
|
||||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
|
||||
license agreements (excluding distributors and resellers) which have been
|
||||
validly granted by You or Your distributors under this License prior to
|
||||
termination shall survive termination.
|
||||
|
||||
6. Disclaimer of Warranty
|
||||
|
||||
Covered Software is provided under this License on an “as is” basis, without
|
||||
warranty of any kind, either expressed, implied, or statutory, including,
|
||||
without limitation, warranties that the Covered Software is free of defects,
|
||||
merchantable, fit for a particular purpose or non-infringing. The entire
|
||||
risk as to the quality and performance of the Covered Software is with You.
|
||||
Should any Covered Software prove defective in any respect, You (not any
|
||||
Contributor) assume the cost of any necessary servicing, repair, or
|
||||
correction. This disclaimer of warranty constitutes an essential part of this
|
||||
License. No use of any Covered Software is authorized under this License
|
||||
except under this disclaimer.
|
||||
|
||||
7. Limitation of Liability
|
||||
|
||||
Under no circumstances and under no legal theory, whether tort (including
|
||||
negligence), contract, or otherwise, shall any Contributor, or anyone who
|
||||
distributes Covered Software as permitted above, be liable to You for any
|
||||
direct, indirect, special, incidental, or consequential damages of any
|
||||
character including, without limitation, damages for lost profits, loss of
|
||||
goodwill, work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses, even if such party shall have been
|
||||
informed of the possibility of such damages. This limitation of liability
|
||||
shall not apply to liability for death or personal injury resulting from such
|
||||
party’s negligence to the extent applicable law prohibits such limitation.
|
||||
Some jurisdictions do not allow the exclusion or limitation of incidental or
|
||||
consequential damages, so this exclusion and limitation may not apply to You.
|
||||
|
||||
8. Litigation
|
||||
|
||||
Any litigation relating to this License may be brought only in the courts of
|
||||
a jurisdiction where the defendant maintains its principal place of business
|
||||
and such litigation shall be governed by laws of that jurisdiction, without
|
||||
reference to its conflict-of-law provisions. Nothing in this Section shall
|
||||
prevent a party’s ability to bring cross-claims or counter-claims.
|
||||
|
||||
9. Miscellaneous
|
||||
|
||||
This License represents the complete agreement concerning the subject matter
|
||||
hereof. If any provision of this License is held to be unenforceable, such
|
||||
provision shall be reformed only to the extent necessary to make it
|
||||
enforceable. Any law or regulation which provides that the language of a
|
||||
contract shall be construed against the drafter shall not be used to construe
|
||||
this License against a Contributor.
|
||||
|
||||
|
||||
10. Versions of the License
|
||||
|
||||
10.1. New Versions
|
||||
|
||||
Mozilla Foundation is the license steward. Except as provided in Section
|
||||
10.3, no one other than the license steward has the right to modify or
|
||||
publish new versions of this License. Each version will be given a
|
||||
distinguishing version number.
|
||||
|
||||
10.2. Effect of New Versions
|
||||
|
||||
You may distribute the Covered Software under the terms of the version of
|
||||
the License under which You originally received the Covered Software, or
|
||||
under the terms of any subsequent version published by the license
|
||||
steward.
|
||||
|
||||
10.3. Modified Versions
|
||||
|
||||
If you create software not governed by this License, and you want to
|
||||
create a new license for such software, you may create and use a modified
|
||||
version of this License if you rename the license and remove any
|
||||
references to the name of the license steward (except to note that such
|
||||
modified license differs from this License).
|
||||
|
||||
10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses
|
||||
If You choose to distribute Source Code Form that is Incompatible With
|
||||
Secondary Licenses under the terms of this version of the License, the
|
||||
notice described in Exhibit B of this License must be attached.
|
||||
|
||||
Exhibit A - Source Code Form License Notice
|
||||
|
||||
This Source Code Form is subject to the
|
||||
terms of the Mozilla Public License, v.
|
||||
2.0. If a copy of the MPL was not
|
||||
distributed with this file, You can
|
||||
obtain one at
|
||||
http://mozilla.org/MPL/2.0/.
|
||||
|
||||
If it is not possible or desirable to put the notice in a particular file, then
|
||||
You may include the notice in a location (such as a LICENSE file in a relevant
|
||||
directory) where a recipient would be likely to look for such a notice.
|
||||
|
||||
You may add additional accurate notices of copyright ownership.
|
||||
|
||||
Exhibit B - “Incompatible With Secondary Licenses” Notice
|
||||
|
||||
This Source Code Form is “Incompatible
|
||||
With Secondary Licenses”, as defined by
|
||||
the Mozilla Public License, v. 2.0.
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
TEST?=./...
|
||||
|
||||
default: test
|
||||
|
||||
# test runs the test suite and vets the code.
|
||||
test: generate
|
||||
@echo "==> Running tests..."
|
||||
@go list $(TEST) \
|
||||
| grep -v "/vendor/" \
|
||||
| xargs -n1 go test -timeout=60s -parallel=10 ${TESTARGS}
|
||||
|
||||
# testrace runs the race checker
|
||||
testrace: generate
|
||||
@echo "==> Running tests (race)..."
|
||||
@go list $(TEST) \
|
||||
| grep -v "/vendor/" \
|
||||
| xargs -n1 go test -timeout=60s -race ${TESTARGS}
|
||||
|
||||
# updatedeps installs all the dependencies needed to run and build.
|
||||
updatedeps:
|
||||
@sh -c "'${CURDIR}/scripts/deps.sh' '${NAME}'"
|
||||
|
||||
# generate runs `go generate` to build the dynamically generated source files.
|
||||
generate:
|
||||
@echo "==> Generating..."
|
||||
@find . -type f -name '.DS_Store' -delete
|
||||
@go list ./... \
|
||||
| grep -v "/vendor/" \
|
||||
| xargs -n1 go generate
|
||||
|
||||
.PHONY: default test testrace updatedeps generate
|
||||
-150
@@ -1,150 +0,0 @@
|
||||
# go-multierror
|
||||
|
||||
[](https://circleci.com/gh/hashicorp/go-multierror)
|
||||
[](https://pkg.go.dev/github.com/hashicorp/go-multierror)
|
||||
|
||||
|
||||
[circleci]: https://app.circleci.com/pipelines/github/hashicorp/go-multierror
|
||||
[godocs]: https://pkg.go.dev/github.com/hashicorp/go-multierror
|
||||
|
||||
`go-multierror` is a package for Go that provides a mechanism for
|
||||
representing a list of `error` values as a single `error`.
|
||||
|
||||
This allows a function in Go to return an `error` that might actually
|
||||
be a list of errors. If the caller knows this, they can unwrap the
|
||||
list and access the errors. If the caller doesn't know, the error
|
||||
formats to a nice human-readable format.
|
||||
|
||||
`go-multierror` is fully compatible with the Go standard library
|
||||
[errors](https://golang.org/pkg/errors/) package, including the
|
||||
functions `As`, `Is`, and `Unwrap`. This provides a standardized approach
|
||||
for introspecting on error values.
|
||||
|
||||
## Installation and Docs
|
||||
|
||||
Install using `go get github.com/hashicorp/go-multierror`.
|
||||
|
||||
Full documentation is available at
|
||||
https://pkg.go.dev/github.com/hashicorp/go-multierror
|
||||
|
||||
### Requires go version 1.13 or newer
|
||||
|
||||
`go-multierror` requires go version 1.13 or newer. Go 1.13 introduced
|
||||
[error wrapping](https://golang.org/doc/go1.13#error_wrapping), which
|
||||
this library takes advantage of.
|
||||
|
||||
If you need to use an earlier version of go, you can use the
|
||||
[v1.0.0](https://github.com/hashicorp/go-multierror/tree/v1.0.0)
|
||||
tag, which doesn't rely on features in go 1.13.
|
||||
|
||||
If you see compile errors that look like the below, it's likely that
|
||||
you're on an older version of go:
|
||||
|
||||
```
|
||||
/go/src/github.com/hashicorp/go-multierror/multierror.go:112:9: undefined: errors.As
|
||||
/go/src/github.com/hashicorp/go-multierror/multierror.go:117:9: undefined: errors.Is
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
go-multierror is easy to use and purposely built to be unobtrusive in
|
||||
existing Go applications/libraries that may not be aware of it.
|
||||
|
||||
**Building a list of errors**
|
||||
|
||||
The `Append` function is used to create a list of errors. This function
|
||||
behaves a lot like the Go built-in `append` function: it doesn't matter
|
||||
if the first argument is nil, a `multierror.Error`, or any other `error`,
|
||||
the function behaves as you would expect.
|
||||
|
||||
```go
|
||||
var result error
|
||||
|
||||
if err := step1(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
if err := step2(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
|
||||
return result
|
||||
```
|
||||
|
||||
**Customizing the formatting of the errors**
|
||||
|
||||
By specifying a custom `ErrorFormat`, you can customize the format
|
||||
of the `Error() string` function:
|
||||
|
||||
```go
|
||||
var result *multierror.Error
|
||||
|
||||
// ... accumulate errors here, maybe using Append
|
||||
|
||||
if result != nil {
|
||||
result.ErrorFormat = func([]error) string {
|
||||
return "errors!"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Accessing the list of errors**
|
||||
|
||||
`multierror.Error` implements `error` so if the caller doesn't know about
|
||||
multierror, it will work just fine. But if you're aware a multierror might
|
||||
be returned, you can use type switches to access the list of errors:
|
||||
|
||||
```go
|
||||
if err := something(); err != nil {
|
||||
if merr, ok := err.(*multierror.Error); ok {
|
||||
// Use merr.Errors
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
You can also use the standard [`errors.Unwrap`](https://golang.org/pkg/errors/#Unwrap)
|
||||
function. This will continue to unwrap into subsequent errors until none exist.
|
||||
|
||||
**Extracting an error**
|
||||
|
||||
The standard library [`errors.As`](https://golang.org/pkg/errors/#As)
|
||||
function can be used directly with a multierror to extract a specific error:
|
||||
|
||||
```go
|
||||
// Assume err is a multierror value
|
||||
err := somefunc()
|
||||
|
||||
// We want to know if "err" has a "RichErrorType" in it and extract it.
|
||||
var errRich RichErrorType
|
||||
if errors.As(err, &errRich) {
|
||||
// It has it, and now errRich is populated.
|
||||
}
|
||||
```
|
||||
|
||||
**Checking for an exact error value**
|
||||
|
||||
Some errors are returned as exact errors such as the [`ErrNotExist`](https://golang.org/pkg/os/#pkg-variables)
|
||||
error in the `os` package. You can check if this error is present by using
|
||||
the standard [`errors.Is`](https://golang.org/pkg/errors/#Is) function.
|
||||
|
||||
```go
|
||||
// Assume err is a multierror value
|
||||
err := somefunc()
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
// err contains os.ErrNotExist
|
||||
}
|
||||
```
|
||||
|
||||
**Returning a multierror only if there are errors**
|
||||
|
||||
If you build a `multierror.Error`, you can use the `ErrorOrNil` function
|
||||
to return an `error` implementation only if there are errors to return:
|
||||
|
||||
```go
|
||||
var result *multierror.Error
|
||||
|
||||
// ... accumulate errors here
|
||||
|
||||
// Return the `error` only if errors were added to the multierror, otherwise
|
||||
// return nil since there are no errors.
|
||||
return result.ErrorOrNil()
|
||||
```
|
||||
-43
@@ -1,43 +0,0 @@
|
||||
package multierror
|
||||
|
||||
// Append is a helper function that will append more errors
|
||||
// onto an Error in order to create a larger multi-error.
|
||||
//
|
||||
// If err is not a multierror.Error, then it will be turned into
|
||||
// one. If any of the errs are multierr.Error, they will be flattened
|
||||
// one level into err.
|
||||
// Any nil errors within errs will be ignored. If err is nil, a new
|
||||
// *Error will be returned.
|
||||
func Append(err error, errs ...error) *Error {
|
||||
switch err := err.(type) {
|
||||
case *Error:
|
||||
// Typed nils can reach here, so initialize if we are nil
|
||||
if err == nil {
|
||||
err = new(Error)
|
||||
}
|
||||
|
||||
// Go through each error and flatten
|
||||
for _, e := range errs {
|
||||
switch e := e.(type) {
|
||||
case *Error:
|
||||
if e != nil {
|
||||
err.Errors = append(err.Errors, e.Errors...)
|
||||
}
|
||||
default:
|
||||
if e != nil {
|
||||
err.Errors = append(err.Errors, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
default:
|
||||
newErrs := make([]error, 0, len(errs)+1)
|
||||
if err != nil {
|
||||
newErrs = append(newErrs, err)
|
||||
}
|
||||
newErrs = append(newErrs, errs...)
|
||||
|
||||
return Append(&Error{}, newErrs...)
|
||||
}
|
||||
}
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
package multierror
|
||||
|
||||
// Flatten flattens the given error, merging any *Errors together into
|
||||
// a single *Error.
|
||||
func Flatten(err error) error {
|
||||
// If it isn't an *Error, just return the error as-is
|
||||
if _, ok := err.(*Error); !ok {
|
||||
return err
|
||||
}
|
||||
|
||||
// Otherwise, make the result and flatten away!
|
||||
flatErr := new(Error)
|
||||
flatten(err, flatErr)
|
||||
return flatErr
|
||||
}
|
||||
|
||||
func flatten(err error, flatErr *Error) {
|
||||
switch err := err.(type) {
|
||||
case *Error:
|
||||
for _, e := range err.Errors {
|
||||
flatten(e, flatErr)
|
||||
}
|
||||
default:
|
||||
flatErr.Errors = append(flatErr.Errors, err)
|
||||
}
|
||||
}
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
package multierror
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ErrorFormatFunc is a function callback that is called by Error to
|
||||
// turn the list of errors into a string.
|
||||
type ErrorFormatFunc func([]error) string
|
||||
|
||||
// ListFormatFunc is a basic formatter that outputs the number of errors
|
||||
// that occurred along with a bullet point list of the errors.
|
||||
func ListFormatFunc(es []error) string {
|
||||
if len(es) == 1 {
|
||||
return fmt.Sprintf("1 error occurred:\n\t* %s\n\n", es[0])
|
||||
}
|
||||
|
||||
points := make([]string, len(es))
|
||||
for i, err := range es {
|
||||
points[i] = fmt.Sprintf("* %s", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"%d errors occurred:\n\t%s\n\n",
|
||||
len(es), strings.Join(points, "\n\t"))
|
||||
}
|
||||
-38
@@ -1,38 +0,0 @@
|
||||
package multierror
|
||||
|
||||
import "sync"
|
||||
|
||||
// Group is a collection of goroutines which return errors that need to be
|
||||
// coalesced.
|
||||
type Group struct {
|
||||
mutex sync.Mutex
|
||||
err *Error
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// Go calls the given function in a new goroutine.
|
||||
//
|
||||
// If the function returns an error it is added to the group multierror which
|
||||
// is returned by Wait.
|
||||
func (g *Group) Go(f func() error) {
|
||||
g.wg.Add(1)
|
||||
|
||||
go func() {
|
||||
defer g.wg.Done()
|
||||
|
||||
if err := f(); err != nil {
|
||||
g.mutex.Lock()
|
||||
g.err = Append(g.err, err)
|
||||
g.mutex.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait blocks until all function calls from the Go method have returned, then
|
||||
// returns the multierror.
|
||||
func (g *Group) Wait() *Error {
|
||||
g.wg.Wait()
|
||||
g.mutex.Lock()
|
||||
defer g.mutex.Unlock()
|
||||
return g.err
|
||||
}
|
||||
-121
@@ -1,121 +0,0 @@
|
||||
package multierror
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Error is an error type to track multiple errors. This is used to
|
||||
// accumulate errors in cases and return them as a single "error".
|
||||
type Error struct {
|
||||
Errors []error
|
||||
ErrorFormat ErrorFormatFunc
|
||||
}
|
||||
|
||||
func (e *Error) Error() string {
|
||||
fn := e.ErrorFormat
|
||||
if fn == nil {
|
||||
fn = ListFormatFunc
|
||||
}
|
||||
|
||||
return fn(e.Errors)
|
||||
}
|
||||
|
||||
// ErrorOrNil returns an error interface if this Error represents
|
||||
// a list of errors, or returns nil if the list of errors is empty. This
|
||||
// function is useful at the end of accumulation to make sure that the value
|
||||
// returned represents the existence of errors.
|
||||
func (e *Error) ErrorOrNil() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
if len(e.Errors) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *Error) GoString() string {
|
||||
return fmt.Sprintf("*%#v", *e)
|
||||
}
|
||||
|
||||
// WrappedErrors returns the list of errors that this Error is wrapping. It is
|
||||
// an implementation of the errwrap.Wrapper interface so that multierror.Error
|
||||
// can be used with that library.
|
||||
//
|
||||
// This method is not safe to be called concurrently. Unlike accessing the
|
||||
// Errors field directly, this function also checks if the multierror is nil to
|
||||
// prevent a null-pointer panic. It satisfies the errwrap.Wrapper interface.
|
||||
func (e *Error) WrappedErrors() []error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.Errors
|
||||
}
|
||||
|
||||
// Unwrap returns an error from Error (or nil if there are no errors).
|
||||
// This error returned will further support Unwrap to get the next error,
|
||||
// etc. The order will match the order of Errors in the multierror.Error
|
||||
// at the time of calling.
|
||||
//
|
||||
// The resulting error supports errors.As/Is/Unwrap so you can continue
|
||||
// to use the stdlib errors package to introspect further.
|
||||
//
|
||||
// This will perform a shallow copy of the errors slice. Any errors appended
|
||||
// to this error after calling Unwrap will not be available until a new
|
||||
// Unwrap is called on the multierror.Error.
|
||||
func (e *Error) Unwrap() error {
|
||||
// If we have no errors then we do nothing
|
||||
if e == nil || len(e.Errors) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If we have exactly one error, we can just return that directly.
|
||||
if len(e.Errors) == 1 {
|
||||
return e.Errors[0]
|
||||
}
|
||||
|
||||
// Shallow copy the slice
|
||||
errs := make([]error, len(e.Errors))
|
||||
copy(errs, e.Errors)
|
||||
return chain(errs)
|
||||
}
|
||||
|
||||
// chain implements the interfaces necessary for errors.Is/As/Unwrap to
|
||||
// work in a deterministic way with multierror. A chain tracks a list of
|
||||
// errors while accounting for the current represented error. This lets
|
||||
// Is/As be meaningful.
|
||||
//
|
||||
// Unwrap returns the next error. In the cleanest form, Unwrap would return
|
||||
// the wrapped error here but we can't do that if we want to properly
|
||||
// get access to all the errors. Instead, users are recommended to use
|
||||
// Is/As to get the correct error type out.
|
||||
//
|
||||
// Precondition: []error is non-empty (len > 0)
|
||||
type chain []error
|
||||
|
||||
// Error implements the error interface
|
||||
func (e chain) Error() string {
|
||||
return e[0].Error()
|
||||
}
|
||||
|
||||
// Unwrap implements errors.Unwrap by returning the next error in the
|
||||
// chain or nil if there are no more errors.
|
||||
func (e chain) Unwrap() error {
|
||||
if len(e) == 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return e[1:]
|
||||
}
|
||||
|
||||
// As implements errors.As by attempting to map to the current value.
|
||||
func (e chain) As(target interface{}) bool {
|
||||
return errors.As(e[0], target)
|
||||
}
|
||||
|
||||
// Is implements errors.Is by comparing the current value directly.
|
||||
func (e chain) Is(target error) bool {
|
||||
return errors.Is(e[0], target)
|
||||
}
|
||||
-37
@@ -1,37 +0,0 @@
|
||||
package multierror
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/hashicorp/errwrap"
|
||||
)
|
||||
|
||||
// Prefix is a helper function that will prefix some text
|
||||
// to the given error. If the error is a multierror.Error, then
|
||||
// it will be prefixed to each wrapped error.
|
||||
//
|
||||
// This is useful to use when appending multiple multierrors
|
||||
// together in order to give better scoping.
|
||||
func Prefix(err error, prefix string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
format := fmt.Sprintf("%s {{err}}", prefix)
|
||||
switch err := err.(type) {
|
||||
case *Error:
|
||||
// Typed nils can reach here, so initialize if we are nil
|
||||
if err == nil {
|
||||
err = new(Error)
|
||||
}
|
||||
|
||||
// Wrap each of the errors
|
||||
for i, e := range err.Errors {
|
||||
err.Errors[i] = errwrap.Wrapf(format, e)
|
||||
}
|
||||
|
||||
return err
|
||||
default:
|
||||
return errwrap.Wrapf(format, err)
|
||||
}
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
package multierror
|
||||
|
||||
// Len implements sort.Interface function for length
|
||||
func (err Error) Len() int {
|
||||
return len(err.Errors)
|
||||
}
|
||||
|
||||
// Swap implements sort.Interface function for swapping elements
|
||||
func (err Error) Swap(i, j int) {
|
||||
err.Errors[i], err.Errors[j] = err.Errors[j], err.Errors[i]
|
||||
}
|
||||
|
||||
// Less implements sort.Interface function for determining order
|
||||
func (err Error) Less(i, j int) bool {
|
||||
return err.Errors[i].Error() < err.Errors[j].Error()
|
||||
}
|
||||
+97
-63
@@ -62,6 +62,7 @@ func emojiCode() map[string]string {
|
||||
":Leo:": "\u264c",
|
||||
":Libra:": "\u264e",
|
||||
":Mrs._Claus:": "\U0001f936",
|
||||
":Mx_Claus:": "\U0001f9d1\u200d\U0001f384",
|
||||
":NEW_button:": "\U0001f195",
|
||||
":NG_button:": "\U0001f196",
|
||||
":OK_button:": "\U0001f197",
|
||||
@@ -220,6 +221,7 @@ func emojiCode() map[string]string {
|
||||
":bald_man:": "\U0001f468\u200d\U0001f9b2",
|
||||
":bald_person:": "\U0001f9d1\u200d\U0001f9b2",
|
||||
":bald_woman:": "\U0001f469\u200d\U0001f9b2",
|
||||
":ballet_dancer:": "\U0001f9d1\u200d\U0001fa70",
|
||||
":ballet_shoes:": "\U0001fa70",
|
||||
":balloon:": "\U0001f388",
|
||||
":ballot_box:": "\U0001f5f3",
|
||||
@@ -764,6 +766,7 @@ func emojiCode() map[string]string {
|
||||
":disappointed_face:": "\U0001f61e",
|
||||
":disappointed_relieved:": "\U0001f625",
|
||||
":disguised_face:": "\U0001f978",
|
||||
":distorted_face:": "\U0001faea",
|
||||
":divide:": "\u2797",
|
||||
":dividers:": "\U0001f5c2",
|
||||
":diving_mask:": "\U0001f93f",
|
||||
@@ -890,6 +893,7 @@ func emojiCode() map[string]string {
|
||||
":face_savoring_food:": "\U0001f60b",
|
||||
":face_screaming_in_fear:": "\U0001f631",
|
||||
":face_vomiting:": "\U0001f92e",
|
||||
":face_with_bags_under_eyes:": "\U0001fae9",
|
||||
":face_with_cowboy_hat:": "\U0001f920",
|
||||
":face_with_crossed-out_eyes:": "\U0001f635",
|
||||
":face_with_diagonal_mouth:": "\U0001fae4",
|
||||
@@ -1018,12 +1022,14 @@ func emojiCode() map[string]string {
|
||||
":ferry:": "\u26f4\ufe0f",
|
||||
":field_hockey:": "\U0001f3d1",
|
||||
":field_hockey_stick_and_ball:": "\U0001f3d1",
|
||||
":fight_cloud:": "\U0001faef",
|
||||
":fiji:": "\U0001f1eb\U0001f1ef",
|
||||
":file_cabinet:": "\U0001f5c4\ufe0f",
|
||||
":file_folder:": "\U0001f4c1",
|
||||
":film_frames:": "\U0001f39e\ufe0f",
|
||||
":film_projector:": "\U0001f4fd\ufe0f",
|
||||
":film_strip:": "\U0001f39e\ufe0f",
|
||||
":fingerprint:": "\U0001fac6",
|
||||
":fingers_crossed:": "\U0001f91e",
|
||||
":fingers_crossed_tone1:": "\U0001f91e\U0001f3fb",
|
||||
":fingers_crossed_tone2:": "\U0001f91e\U0001f3fc",
|
||||
@@ -1251,6 +1257,7 @@ func emojiCode() map[string]string {
|
||||
":flag-rs:": "\U0001f1f7\U0001f1f8",
|
||||
":flag-rw:": "\U0001f1f7\U0001f1fc",
|
||||
":flag-sa:": "\U0001f1f8\U0001f1e6",
|
||||
":flag-sark:": "\U0001f1e8\U0001f1f6",
|
||||
":flag-sb:": "\U0001f1f8\U0001f1e7",
|
||||
":flag-sc:": "\U0001f1f8\U0001f1e8",
|
||||
":flag-scotland:": "\U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f",
|
||||
@@ -1504,6 +1511,7 @@ func emojiCode() map[string]string {
|
||||
":flag_Réunion:": "\U0001f1f7\U0001f1ea",
|
||||
":flag_Samoa:": "\U0001f1fc\U0001f1f8",
|
||||
":flag_San_Marino:": "\U0001f1f8\U0001f1f2",
|
||||
":flag_Sark:": "\U0001f1e8\U0001f1f6",
|
||||
":flag_Saudi_Arabia:": "\U0001f1f8\U0001f1e6",
|
||||
":flag_Scotland:": "\U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f",
|
||||
":flag_Senegal:": "\U0001f1f8\U0001f1f3",
|
||||
@@ -1995,6 +2003,7 @@ func emojiCode() map[string]string {
|
||||
":haircut:": "\U0001f487\u200d\u2640\ufe0f",
|
||||
":haircut_man:": "\U0001f487\u200d\u2642\ufe0f",
|
||||
":haircut_woman:": "\U0001f487\u200d\u2640\ufe0f",
|
||||
":hairy_creature:": "\U0001fac8",
|
||||
":haiti:": "\U0001f1ed\U0001f1f9",
|
||||
":hamburger:": "\U0001f354",
|
||||
":hammer:": "\U0001f528",
|
||||
@@ -2012,43 +2021,44 @@ func emojiCode() map[string]string {
|
||||
":hand_splayed_tone5:": "\U0001f590\U0001f3ff",
|
||||
":hand_with_fingers_splayed:": "\U0001f590",
|
||||
":hand_with_index_finger_and_thumb_crossed:": "\U0001faf0",
|
||||
":handbag:": "\U0001f45c",
|
||||
":handball:": "\U0001f93e",
|
||||
":handball_person:": "\U0001f93e",
|
||||
":handshake:": "\U0001f91d",
|
||||
":hankey:": "\U0001f4a9",
|
||||
":hash:": "#\ufe0f\u20e3",
|
||||
":hatched_chick:": "\U0001f425",
|
||||
":hatching_chick:": "\U0001f423",
|
||||
":head_bandage:": "\U0001f915",
|
||||
":head_shaking_horizontally:": "\U0001f642\u200d\u2194\ufe0f",
|
||||
":head_shaking_vertically:": "\U0001f642\u200d\u2195\ufe0f",
|
||||
":headphone:": "\U0001f3a7",
|
||||
":headphones:": "\U0001f3a7",
|
||||
":headstone:": "\U0001faa6",
|
||||
":health_worker:": "\U0001f9d1\u200d\u2695\ufe0f",
|
||||
":hear-no-evil_monkey:": "\U0001f649",
|
||||
":hear_no_evil:": "\U0001f649",
|
||||
":heard_mcdonald_islands:": "\U0001f1ed\U0001f1f2",
|
||||
":heart:": "\u2764\ufe0f",
|
||||
":heart_decoration:": "\U0001f49f",
|
||||
":heart_exclamation:": "\u2763",
|
||||
":heart_eyes:": "\U0001f60d",
|
||||
":heart_eyes_cat:": "\U0001f63b",
|
||||
":heart_hands:": "\U0001faf6",
|
||||
":heart_on_fire:": "\u2764\ufe0f\u200d\U0001f525",
|
||||
":heart_suit:": "\u2665",
|
||||
":heart_with_arrow:": "\U0001f498",
|
||||
":heart_with_ribbon:": "\U0001f49d",
|
||||
":heartbeat:": "\U0001f493",
|
||||
":heartpulse:": "\U0001f497",
|
||||
":hearts:": "\u2665\ufe0f",
|
||||
":heavy_check_mark:": "\u2714\ufe0f",
|
||||
":heavy_division_sign:": "\u2797",
|
||||
":heavy_dollar_sign:": "\U0001f4b2",
|
||||
":heavy_equals_sign:": "\U0001f7f0",
|
||||
":heavy_exclamation_mark:": "\u2757",
|
||||
":heavy_heart_exclamation:": "\u2763\ufe0f",
|
||||
":handbag:": "\U0001f45c",
|
||||
":handball:": "\U0001f93e",
|
||||
":handball_person:": "\U0001f93e",
|
||||
":handshake:": "\U0001f91d",
|
||||
":hankey:": "\U0001f4a9",
|
||||
":harp:": "\U0001fa89",
|
||||
":hash:": "#\ufe0f\u20e3",
|
||||
":hatched_chick:": "\U0001f425",
|
||||
":hatching_chick:": "\U0001f423",
|
||||
":head_bandage:": "\U0001f915",
|
||||
":head_shaking_horizontally:": "\U0001f642\u200d\u2194\ufe0f",
|
||||
":head_shaking_vertically:": "\U0001f642\u200d\u2195\ufe0f",
|
||||
":headphone:": "\U0001f3a7",
|
||||
":headphones:": "\U0001f3a7",
|
||||
":headstone:": "\U0001faa6",
|
||||
":health_worker:": "\U0001f9d1\u200d\u2695\ufe0f",
|
||||
":hear-no-evil_monkey:": "\U0001f649",
|
||||
":hear_no_evil:": "\U0001f649",
|
||||
":heard_mcdonald_islands:": "\U0001f1ed\U0001f1f2",
|
||||
":heart:": "\u2764\ufe0f",
|
||||
":heart_decoration:": "\U0001f49f",
|
||||
":heart_exclamation:": "\u2763",
|
||||
":heart_eyes:": "\U0001f60d",
|
||||
":heart_eyes_cat:": "\U0001f63b",
|
||||
":heart_hands:": "\U0001faf6",
|
||||
":heart_on_fire:": "\u2764\ufe0f\u200d\U0001f525",
|
||||
":heart_suit:": "\u2665",
|
||||
":heart_with_arrow:": "\U0001f498",
|
||||
":heart_with_ribbon:": "\U0001f49d",
|
||||
":heartbeat:": "\U0001f493",
|
||||
":heartpulse:": "\U0001f497",
|
||||
":hearts:": "\u2665\ufe0f",
|
||||
":heavy_check_mark:": "\u2714\ufe0f",
|
||||
":heavy_division_sign:": "\u2797",
|
||||
":heavy_dollar_sign:": "\U0001f4b2",
|
||||
":heavy_equals_sign:": "\U0001f7f0",
|
||||
":heavy_exclamation_mark:": "\u2757",
|
||||
":heavy_heart_exclamation:": "\u2763\ufe0f",
|
||||
":heavy_heart_exclamation_mark_ornament:": "\u2763\ufe0f",
|
||||
":heavy_minus_sign:": "\u2796",
|
||||
":heavy_multiplication_x:": "\u2716\ufe0f",
|
||||
@@ -2237,6 +2247,7 @@ func emojiCode() map[string]string {
|
||||
":ladder:": "\U0001fa9c",
|
||||
":lady_beetle:": "\U0001f41e",
|
||||
":ladybug:": "\U0001f41e",
|
||||
":landslide:": "\U0001f6d8",
|
||||
":lantern:": "\U0001f3ee",
|
||||
":laos:": "\U0001f1f1\U0001f1e6",
|
||||
":laptop:": "\U0001f4bb",
|
||||
@@ -2263,6 +2274,7 @@ func emojiCode() map[string]string {
|
||||
":latvia:": "\U0001f1f1\U0001f1fb",
|
||||
":laughing:": "\U0001f606",
|
||||
":leaf_fluttering_in_wind:": "\U0001f343",
|
||||
":leafless_tree:": "\U0001fabe",
|
||||
":leafy_green:": "\U0001f96c",
|
||||
":leaves:": "\U0001f343",
|
||||
":lebanon:": "\U0001f1f1\U0001f1e7",
|
||||
@@ -3146,6 +3158,7 @@ func emojiCode() map[string]string {
|
||||
":orange_heart:": "\U0001f9e1",
|
||||
":orange_square:": "\U0001f7e7",
|
||||
":orangutan:": "\U0001f9a7",
|
||||
":orca:": "\U0001facd",
|
||||
":orthodox_cross:": "\u2626\ufe0f",
|
||||
":otter:": "\U0001f9a6",
|
||||
":outbox_tray:": "\U0001f4e4",
|
||||
@@ -3702,6 +3715,7 @@ func emojiCode() map[string]string {
|
||||
":rolling_on_the_floor_laughing:": "\U0001f923",
|
||||
":romania:": "\U0001f1f7\U0001f1f4",
|
||||
":rooster:": "\U0001f413",
|
||||
":root_vegetable:": "\U0001fadc",
|
||||
":rose:": "\U0001f339",
|
||||
":rosette:": "\U0001f3f5\ufe0f",
|
||||
":rotating_light:": "\U0001f6a8",
|
||||
@@ -3810,6 +3824,7 @@ func emojiCode() map[string]string {
|
||||
":shopping_trolley:": "\U0001f6d2",
|
||||
":shortcake:": "\U0001f370",
|
||||
":shorts:": "\U0001fa73",
|
||||
":shovel:": "\U0001fa8f",
|
||||
":shower:": "\U0001f6bf",
|
||||
":shrimp:": "\U0001f990",
|
||||
":shrug:": "\U0001f937",
|
||||
@@ -3938,6 +3953,7 @@ func emojiCode() map[string]string {
|
||||
":spiral_note_pad:": "\U0001f5d2\ufe0f",
|
||||
":spiral_notepad:": "\U0001f5d2",
|
||||
":spiral_shell:": "\U0001f41a",
|
||||
":splatter:": "\U0001fadf",
|
||||
":spock-hand:": "\U0001f596",
|
||||
":sponge:": "\U0001f9fd",
|
||||
":spoon:": "\U0001f944",
|
||||
@@ -4129,6 +4145,7 @@ func emojiCode() map[string]string {
|
||||
":tram_car:": "\U0001f68b",
|
||||
":transgender_flag:": "\U0001f3f3\ufe0f\u200d\u26a7\ufe0f",
|
||||
":transgender_symbol:": "\u26a7\ufe0f",
|
||||
":treasure_chest:": "\U0001fa8e",
|
||||
":triangular_flag:": "\U0001f6a9",
|
||||
":triangular_flag_on_post:": "\U0001f6a9",
|
||||
":triangular_ruler:": "\U0001f4d0",
|
||||
@@ -4139,6 +4156,7 @@ func emojiCode() map[string]string {
|
||||
":triumph:": "\U0001f624",
|
||||
":troll:": "\U0001f9cc",
|
||||
":trolleybus:": "\U0001f68e",
|
||||
":trombone:": "\U0001fa8a",
|
||||
":trophy:": "\U0001f3c6",
|
||||
":tropical_drink:": "\U0001f379",
|
||||
":tropical_fish:": "\U0001f420",
|
||||
@@ -4881,6 +4899,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f1e8\U0001f1f3": {":cn:", ":flag_cn:", ":flag_China:"},
|
||||
"\U0001f1e8\U0001f1f4": {":flag-co:", ":flag_co:", ":colombia:", ":flag_Colombia:"},
|
||||
"\U0001f1e8\U0001f1f5": {":flag-cp:", ":flag_cp:", ":clipperton_island:", ":flag_Clipperton_Island:"},
|
||||
"\U0001f1e8\U0001f1f6": {":flag-sark:", ":flag_Sark:"},
|
||||
"\U0001f1e8\U0001f1f7": {":flag-cr:", ":flag_cr:", ":costa_rica:", ":flag_Costa_Rica:"},
|
||||
"\U0001f1e8\U0001f1fa": {":cuba:", ":flag-cu:", ":flag_cu:", ":flag_Cuba:"},
|
||||
"\U0001f1e8\U0001f1fb": {":flag-cv:", ":flag_cv:", ":cape_verde:", ":flag_Cape_Verde:"},
|
||||
@@ -5246,7 +5265,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f381": {":gift:", ":wrapped_gift:"},
|
||||
"\U0001f382": {":birthday:", ":birthday_cake:"},
|
||||
"\U0001f383": {":jack-o-lantern:", ":jack_o_lantern:"},
|
||||
"\U0001f384": {":Christmas_tree:", ":christmas_tree:"},
|
||||
"\U0001f384": {":christmas_tree:", ":Christmas_tree:"},
|
||||
"\U0001f385": {":santa:", ":Santa_Claus:"},
|
||||
"\U0001f385\U0001f3fb": {":santa_tone1:"},
|
||||
"\U0001f385\U0001f3fc": {":santa_tone2:"},
|
||||
@@ -5465,7 +5484,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f3ec": {":department_store:"},
|
||||
"\U0001f3ed": {":factory:"},
|
||||
"\U0001f3ee": {":lantern:", ":izakaya_lantern:", ":red_paper_lantern:"},
|
||||
"\U0001f3ef": {":Japanese_castle:", ":japanese_castle:"},
|
||||
"\U0001f3ef": {":japanese_castle:", ":Japanese_castle:"},
|
||||
"\U0001f3f0": {":castle:", ":european_castle:"},
|
||||
"\U0001f3f3": {":flag_white:", ":white_flag:"},
|
||||
"\U0001f3f3\ufe0f": {":waving_white_flag:"},
|
||||
@@ -5608,7 +5627,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f44b\U0001f3fd": {":wave_tone3:"},
|
||||
"\U0001f44b\U0001f3fe": {":wave_tone4:"},
|
||||
"\U0001f44b\U0001f3ff": {":wave_tone5:"},
|
||||
"\U0001f44c": {":OK_hand:", ":ok_hand:"},
|
||||
"\U0001f44c": {":ok_hand:", ":OK_hand:"},
|
||||
"\U0001f44c\U0001f3fb": {":ok_hand_tone1:"},
|
||||
"\U0001f44c\U0001f3fc": {":ok_hand_tone2:"},
|
||||
"\U0001f44c\U0001f3fd": {":ok_hand_tone3:"},
|
||||
@@ -6169,7 +6188,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f4a1": {":bulb:", ":light_bulb:"},
|
||||
"\U0001f4a2": {":anger:", ":anger_symbol:"},
|
||||
"\U0001f4a3": {":bomb:"},
|
||||
"\U0001f4a4": {":ZZZ:", ":zzz:"},
|
||||
"\U0001f4a4": {":zzz:", ":ZZZ:"},
|
||||
"\U0001f4a5": {":boom:", ":collision:"},
|
||||
"\U0001f4a6": {":sweat_drops:", ":sweat_droplets:"},
|
||||
"\U0001f4a7": {":droplet:"},
|
||||
@@ -6466,8 +6485,8 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f5fa": {":map:"},
|
||||
"\U0001f5fa\ufe0f": {":world_map:"},
|
||||
"\U0001f5fb": {":mount_fuji:"},
|
||||
"\U0001f5fc": {":Tokyo_tower:", ":tokyo_tower:"},
|
||||
"\U0001f5fd": {":Statue_of_Liberty:", ":statue_of_liberty:"},
|
||||
"\U0001f5fc": {":tokyo_tower:", ":Tokyo_tower:"},
|
||||
"\U0001f5fd": {":statue_of_liberty:", ":Statue_of_Liberty:"},
|
||||
"\U0001f5fe": {":japan:", ":map_of_Japan:"},
|
||||
"\U0001f5ff": {":moai:", ":moyai:"},
|
||||
"\U0001f600": {":grinning:", ":grinning_face:"},
|
||||
@@ -6544,7 +6563,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f642\u200d\u2195\ufe0f": {":head_shaking_vertically:"},
|
||||
"\U0001f643": {":upside_down:", ":upside-down_face:", ":upside_down_face:"},
|
||||
"\U0001f644": {":roll_eyes:", ":rolling_eyes:", ":face_with_rolling_eyes:"},
|
||||
"\U0001f645": {":person_gesturing_NO:", ":person_gesturing_no:"},
|
||||
"\U0001f645": {":person_gesturing_no:", ":person_gesturing_NO:"},
|
||||
"\U0001f645\U0001f3fb": {":person_gesturing_no_tone1:"},
|
||||
"\U0001f645\U0001f3fb\u200d\u2640\ufe0f": {":woman_gesturing_no_tone1:"},
|
||||
"\U0001f645\U0001f3fb\u200d\u2642\ufe0f": {":man_gesturing_no_tone1:"},
|
||||
@@ -6560,9 +6579,9 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f645\U0001f3ff": {":person_gesturing_no_tone5:"},
|
||||
"\U0001f645\U0001f3ff\u200d\u2640\ufe0f": {":woman_gesturing_no_tone5:"},
|
||||
"\U0001f645\U0001f3ff\u200d\u2642\ufe0f": {":man_gesturing_no_tone5:"},
|
||||
"\U0001f645\u200d\u2640\ufe0f": {":no_good:", ":ng_woman:", ":no_good_woman:", ":woman-gesturing-no:", ":woman_gesturing_NO:", ":woman_gesturing_no:"},
|
||||
"\U0001f645\u200d\u2642\ufe0f": {":ng_man:", ":no_good_man:", ":man-gesturing-no:", ":man_gesturing_NO:", ":man_gesturing_no:"},
|
||||
"\U0001f646": {":ok_person:", ":person_gesturing_OK:", ":person_gesturing_ok:"},
|
||||
"\U0001f645\u200d\u2640\ufe0f": {":no_good:", ":ng_woman:", ":no_good_woman:", ":woman-gesturing-no:", ":woman_gesturing_no:", ":woman_gesturing_NO:"},
|
||||
"\U0001f645\u200d\u2642\ufe0f": {":ng_man:", ":no_good_man:", ":man-gesturing-no:", ":man_gesturing_no:", ":man_gesturing_NO:"},
|
||||
"\U0001f646": {":ok_person:", ":person_gesturing_ok:", ":person_gesturing_OK:"},
|
||||
"\U0001f646\U0001f3fb": {":person_gesturing_ok_tone1:"},
|
||||
"\U0001f646\U0001f3fb\u200d\u2640\ufe0f": {":woman_gesturing_ok_tone1:"},
|
||||
"\U0001f646\U0001f3fb\u200d\u2642\ufe0f": {":man_gesturing_ok_tone1:"},
|
||||
@@ -6578,8 +6597,8 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f646\U0001f3ff": {":person_gesturing_ok_tone5:"},
|
||||
"\U0001f646\U0001f3ff\u200d\u2640\ufe0f": {":woman_gesturing_ok_tone5:"},
|
||||
"\U0001f646\U0001f3ff\u200d\u2642\ufe0f": {":man_gesturing_ok_tone5:"},
|
||||
"\U0001f646\u200d\u2640\ufe0f": {":ok_woman:", ":woman-gesturing-ok:", ":woman_gesturing_OK:", ":woman_gesturing_ok:"},
|
||||
"\U0001f646\u200d\u2642\ufe0f": {":ok_man:", ":man-gesturing-ok:", ":man_gesturing_OK:", ":man_gesturing_ok:"},
|
||||
"\U0001f646\u200d\u2640\ufe0f": {":ok_woman:", ":woman-gesturing-ok:", ":woman_gesturing_ok:", ":woman_gesturing_OK:"},
|
||||
"\U0001f646\u200d\u2642\ufe0f": {":ok_man:", ":man-gesturing-ok:", ":man_gesturing_ok:", ":man_gesturing_OK:"},
|
||||
"\U0001f647": {":bow:", ":person_bowing:"},
|
||||
"\U0001f647\U0001f3fb": {":person_bowing_tone1:"},
|
||||
"\U0001f647\U0001f3fb\u200d\u2640\ufe0f": {":woman_bowing_tone1:"},
|
||||
@@ -6831,6 +6850,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f6d5": {":hindu_temple:"},
|
||||
"\U0001f6d6": {":hut:"},
|
||||
"\U0001f6d7": {":elevator:"},
|
||||
"\U0001f6d8": {":landslide:"},
|
||||
"\U0001f6dc": {":wireless:"},
|
||||
"\U0001f6dd": {":playground_slide:"},
|
||||
"\U0001f6de": {":wheel:"},
|
||||
@@ -7186,7 +7206,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f993": {":zebra:", ":zebra_face:"},
|
||||
"\U0001f994": {":hedgehog:"},
|
||||
"\U0001f995": {":sauropod:"},
|
||||
"\U0001f996": {":T-Rex:", ":t-rex:", ":t_rex:"},
|
||||
"\U0001f996": {":t-rex:", ":T-Rex:", ":t_rex:"},
|
||||
"\U0001f997": {":cricket:"},
|
||||
"\U0001f998": {":kangaroo:"},
|
||||
"\U0001f999": {":llama:"},
|
||||
@@ -7267,7 +7287,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f9d1\u200d\U0001f33e": {":farmer:"},
|
||||
"\U0001f9d1\u200d\U0001f373": {":cook:"},
|
||||
"\U0001f9d1\u200d\U0001f37c": {":person_feeding_baby:"},
|
||||
"\U0001f9d1\u200d\U0001f384": {":mx_claus:"},
|
||||
"\U0001f9d1\u200d\U0001f384": {":mx_claus:", ":Mx_Claus:"},
|
||||
"\U0001f9d1\u200d\U0001f393": {":student:"},
|
||||
"\U0001f9d1\u200d\U0001f3a4": {":singer:"},
|
||||
"\U0001f9d1\u200d\U0001f3a8": {":artist:"},
|
||||
@@ -7294,6 +7314,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001f9d1\u200d\U0001f9d1\u200d\U0001f9d2\u200d\U0001f9d2": {":family_adult_adult_child_child:"},
|
||||
"\U0001f9d1\u200d\U0001f9d2": {":family_adult_child:"},
|
||||
"\U0001f9d1\u200d\U0001f9d2\u200d\U0001f9d2": {":family_adult_child_child:"},
|
||||
"\U0001f9d1\u200d\U0001fa70": {":ballet_dancer:"},
|
||||
"\U0001f9d1\u200d\u2695\ufe0f": {":health_worker:"},
|
||||
"\U0001f9d1\u200d\u2696\ufe0f": {":judge:"},
|
||||
"\U0001f9d1\u200d\u2708\ufe0f": {":pilot:"},
|
||||
@@ -7520,6 +7541,10 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001fa86": {":nesting_dolls:"},
|
||||
"\U0001fa87": {":maracas:"},
|
||||
"\U0001fa88": {":flute:"},
|
||||
"\U0001fa89": {":harp:"},
|
||||
"\U0001fa8a": {":trombone:"},
|
||||
"\U0001fa8e": {":treasure_chest:"},
|
||||
"\U0001fa8f": {":shovel:"},
|
||||
"\U0001fa90": {":ringed_planet:"},
|
||||
"\U0001fa91": {":chair:"},
|
||||
"\U0001fa92": {":razor:"},
|
||||
@@ -7566,6 +7591,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001fabb": {":hyacinth:"},
|
||||
"\U0001fabc": {":jellyfish:"},
|
||||
"\U0001fabd": {":wing:"},
|
||||
"\U0001fabe": {":leafless_tree:"},
|
||||
"\U0001fabf": {":goose:"},
|
||||
"\U0001fac0": {":anatomical_heart:"},
|
||||
"\U0001fac1": {":lungs:"},
|
||||
@@ -7573,6 +7599,9 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001fac3": {":pregnant_man:"},
|
||||
"\U0001fac4": {":pregnant_person:"},
|
||||
"\U0001fac5": {":person_with_crown:"},
|
||||
"\U0001fac6": {":fingerprint:"},
|
||||
"\U0001fac8": {":hairy_creature:"},
|
||||
"\U0001facd": {":orca:"},
|
||||
"\U0001face": {":moose:"},
|
||||
"\U0001facf": {":donkey:"},
|
||||
"\U0001fad0": {":blueberries:"},
|
||||
@@ -7587,6 +7616,8 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001fad9": {":jar:"},
|
||||
"\U0001fada": {":ginger_root:"},
|
||||
"\U0001fadb": {":pea_pod:"},
|
||||
"\U0001fadc": {":root_vegetable:"},
|
||||
"\U0001fadf": {":splatter:"},
|
||||
"\U0001fae0": {":melting_face:"},
|
||||
"\U0001fae1": {":saluting_face:"},
|
||||
"\U0001fae2": {":face_with_open_eyes_and_hand_over_mouth:"},
|
||||
@@ -7596,6 +7627,9 @@ func emojiRevCode() map[string][]string {
|
||||
"\U0001fae6": {":biting_lip:"},
|
||||
"\U0001fae7": {":bubbles:"},
|
||||
"\U0001fae8": {":shaking_face:"},
|
||||
"\U0001fae9": {":face_with_bags_under_eyes:"},
|
||||
"\U0001faea": {":distorted_face:"},
|
||||
"\U0001faef": {":fight_cloud:"},
|
||||
"\U0001faf0": {":hand_with_index_finger_and_thumb_crossed:"},
|
||||
"\U0001faf1": {":rightwards_hand:"},
|
||||
"\U0001faf2": {":leftwards_hand:"},
|
||||
@@ -7709,18 +7743,18 @@ func emojiRevCode() map[string][]string {
|
||||
"\u263a\ufe0f": {":relaxed:"},
|
||||
"\u2640\ufe0f": {":female_sign:"},
|
||||
"\u2642\ufe0f": {":male_sign:"},
|
||||
"\u2648": {":Aries:", ":aries:"},
|
||||
"\u2649": {":Taurus:", ":taurus:"},
|
||||
"\u264a": {":Gemini:", ":gemini:"},
|
||||
"\u264b": {":Cancer:", ":cancer:"},
|
||||
"\u264c": {":Leo:", ":leo:"},
|
||||
"\u264d": {":Virgo:", ":virgo:"},
|
||||
"\u264e": {":Libra:", ":libra:"},
|
||||
"\u2648": {":aries:", ":Aries:"},
|
||||
"\u2649": {":taurus:", ":Taurus:"},
|
||||
"\u264a": {":gemini:", ":Gemini:"},
|
||||
"\u264b": {":cancer:", ":Cancer:"},
|
||||
"\u264c": {":leo:", ":Leo:"},
|
||||
"\u264d": {":virgo:", ":Virgo:"},
|
||||
"\u264e": {":libra:", ":Libra:"},
|
||||
"\u264f": {":Scorpio:", ":scorpius:"},
|
||||
"\u2650": {":Sagittarius:", ":sagittarius:"},
|
||||
"\u2651": {":Capricorn:", ":capricorn:"},
|
||||
"\u2652": {":Aquarius:", ":aquarius:"},
|
||||
"\u2653": {":Pisces:", ":pisces:"},
|
||||
"\u2650": {":sagittarius:", ":Sagittarius:"},
|
||||
"\u2651": {":capricorn:", ":Capricorn:"},
|
||||
"\u2652": {":aquarius:", ":Aquarius:"},
|
||||
"\u2653": {":pisces:", ":Pisces:"},
|
||||
"\u265f\ufe0f": {":chess_pawn:"},
|
||||
"\u2660": {":spade_suit:"},
|
||||
"\u2660\ufe0f": {":spades:"},
|
||||
@@ -7763,7 +7797,7 @@ func emojiRevCode() map[string][]string {
|
||||
"\u26c5": {":partly_sunny:", ":sun_behind_cloud:"},
|
||||
"\u26c8": {":thunder_cloud_rain:", ":cloud_with_lightning_and_rain:"},
|
||||
"\u26c8\ufe0f": {":thunder_cloud_and_rain:"},
|
||||
"\u26ce": {":Ophiuchus:", ":ophiuchus:"},
|
||||
"\u26ce": {":ophiuchus:", ":Ophiuchus:"},
|
||||
"\u26cf\ufe0f": {":pick:"},
|
||||
"\u26d1": {":helmet_with_cross:", ":rescue_worker’s_helmet:"},
|
||||
"\u26d1\ufe0f": {":rescue_worker_helmet:", ":helmet_with_white_cross:"},
|
||||
|
||||
@@ -17,3 +17,12 @@ maintenanceNotifications/docs/
|
||||
# Docker-generated files (TLS certificates, cluster data, etc.)
|
||||
dockers/*/tls/
|
||||
dockers/osscluster-tls/
|
||||
|
||||
# Per-user Claude Code settings (machine/personal overrides, not shared policy)
|
||||
.claude/settings.local.json
|
||||
|
||||
# example build artifacts
|
||||
example/autopipeline/autopipeline
|
||||
|
||||
# Claude Code worktrees (ephemeral, local)
|
||||
.claude/worktrees/
|
||||
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for AI coding agents (and humans) working in this repository. This is
|
||||
the shared, tool-agnostic source of truth: Claude Code loads it through
|
||||
`CLAUDE.md` (which imports this file), and other agents (Codex, Cursor, Aider,
|
||||
Zed, …) read `AGENTS.md` directly. Edit repository guidance here, not in
|
||||
`CLAUDE.md`.
|
||||
|
||||
## Repository
|
||||
|
||||
go-redis is the official Redis client for Go. Module path:
|
||||
`github.com/redis/go-redis/v9` (Go 1.24+). The repo is a multi-module workspace
|
||||
— every directory containing a `go.mod` is built and tested independently:
|
||||
|
||||
- root (`github.com/redis/go-redis/v9`) — the client library.
|
||||
- `extra/redisotel`, `extra/redisotel-native`, `extra/redisprometheus`,
|
||||
`extra/rediscensus`, `extra/rediscmd` — instrumentation adapters with their
|
||||
own module paths (so they can pin large telemetry deps without forcing them on
|
||||
root consumers).
|
||||
- `internal/customvet` — custom `go vet` analyzers (also its own module).
|
||||
- `maintnotifications/e2e`, `doctests`, `fuzz`, examples under `example/` —
|
||||
separate modules.
|
||||
|
||||
The Makefile iterates over every `go.mod` (`GO_MOD_DIRS`) when running
|
||||
`test.ci`, `go_mod_tidy`, etc. When you add a dependency in one module, you
|
||||
almost never need to update the others.
|
||||
|
||||
## Common commands
|
||||
|
||||
Tests run against a Redis stack started via Docker Compose. Profiles in
|
||||
`docker-compose.yml` control which services come up (`standalone`, `cluster`,
|
||||
`sentinel`, `all`, `e2e`).
|
||||
|
||||
```sh
|
||||
make docker.start # bring up the full test stack (profile: all)
|
||||
make docker.stop
|
||||
make test # docker.start -> test.ci -> docker.stop
|
||||
make test.ci # run tests assuming containers are already up
|
||||
make test.ci.skip-vectorsets # when REDIS_VERSION < 8
|
||||
make bench # go test -bench=. (root module only)
|
||||
make fmt # gofumpt + goimports -local github.com/redis/go-redis
|
||||
make build
|
||||
make go_mod_tidy # go mod tidy across every module
|
||||
```
|
||||
|
||||
E2E (maintenance notifications) needs the extra `cae-resp-proxy` service:
|
||||
|
||||
```sh
|
||||
make test.e2e # starts e2e profile, runs ./maintnotifications/e2e/, tears down
|
||||
make test.e2e.docker # subset that runs inside docker
|
||||
make test.e2e.logic # logic-only tests, no proxy required
|
||||
```
|
||||
|
||||
Run a single test. The root suite is Ginkgo-based (`bsm/ginkgo` + `bsm/gomega`
|
||||
forks), so `go test -run` matches the Go-level wrapper and you focus a spec with
|
||||
the Ginkgo flag:
|
||||
|
||||
```sh
|
||||
go test -run TestGinkgoSuite . -ginkgo.focus="ZAdd"
|
||||
go test -run TestGinkgoSuite . -ginkgo.focus="cluster"
|
||||
```
|
||||
|
||||
Plain `go test` tests (most files outside the Ginkgo suite, e.g. `internal/...`,
|
||||
`maintnotifications/...`) work the usual way:
|
||||
|
||||
```sh
|
||||
go test -run TestConnStateMachine ./internal/pool/...
|
||||
go test -race -run TestCircuitBreaker ./maintnotifications/...
|
||||
```
|
||||
|
||||
Env knobs (passed through the Makefile):
|
||||
|
||||
- `REDIS_VERSION` — e.g. `8.8`. Drives both the test image tag and
|
||||
`main_test.go` version-gating (`SkipBeforeRedisVersion` /
|
||||
`SkipAfterRedisVersion`).
|
||||
- `CLIENT_LIBS_TEST_IMAGE` — full image ref, e.g.
|
||||
`redislabs/client-libs-test:8.8-m03`.
|
||||
- `RE_CLUSTER=true` — run against a Redis Enterprise cluster instead of the
|
||||
docker-compose stack (the suite then skips ring/sentinel/TLS-cluster setup).
|
||||
- `RCE_DOCKER=true` — Redis CE in docker (default for `make test`).
|
||||
- `REDIS_PORT` — override the default standalone port (`6380`).
|
||||
|
||||
CI also runs the custom vet tool:
|
||||
`go vet -vettool ./internal/customvet/customvet ./...`. The `setval` analyzer
|
||||
requires every `Cmder` with a `Result()` to also have a `SetVal()`.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Client types (root package)
|
||||
|
||||
All clients are in the root package and share most plumbing:
|
||||
|
||||
- `Client` (`redis.go`) — single-node client.
|
||||
- `ClusterClient` (`osscluster.go`) — Redis Cluster aware. `osscluster_router.go`
|
||||
routes commands to the right shard; `internal/routing/` handles cluster-wide
|
||||
aggregation policies (e.g. fan-out for `KEYS`, `DBSIZE`).
|
||||
- `Ring` (`ring.go`) — client-side sharding across independent Redis nodes
|
||||
(consistent hashing, no cluster protocol).
|
||||
- Failover client (`sentinel.go`) — Sentinel-managed failover.
|
||||
- `UniversalClient` (`universal.go`) — wrapper that picks one of the above based
|
||||
on options.
|
||||
|
||||
Command surface lives in topical files: `string_commands.go`, `hash_commands.go`,
|
||||
`stream_commands.go`, `search_commands.go`, `vectorset_commands.go`, etc. Each
|
||||
file defines methods on the shared `Cmdable` interface so every client type gets
|
||||
the same API.
|
||||
|
||||
### Hooks (`redis.go` `hooksMixin`)
|
||||
|
||||
Three hook chains run around every operation: `DialHook`, `ProcessHook`,
|
||||
`ProcessPipelineHook`. Hooks are registered via `client.AddHook(...)` and chain
|
||||
in FIFO order; each hook must call `next` to continue. When a hook wraps an
|
||||
error, it must call `cmd.SetErr(wrappedErr)` so the typed-error helpers
|
||||
(`redis.IsLoadingError`, `IsMovedError`, etc. in `error.go`) keep working through
|
||||
`errors.As`. The README has a longer pipeline-hook example.
|
||||
|
||||
### Connection pool (`internal/pool`)
|
||||
|
||||
Owns dialing, idle/active connection bookkeeping, conn state (`conn_state.go`),
|
||||
pubsub-conn lifecycle (`pubsub.go`), and the dial-retry/backoff logic that powers
|
||||
`DialerRetries` / `DialerRetryBackoff` (also exposed at `dial_retry_backoff.go`
|
||||
in the root). `OnConnect`, `MinIdleConns`, and the buffer-size options
|
||||
(`ReadBufferSize`/`WriteBufferSize`, default 32 KiB since v9.12) flow through
|
||||
here.
|
||||
|
||||
### Protocol (`internal/proto`)
|
||||
|
||||
RESP2/RESP3 reader and writer. Push notifications (RESP3 `>`-prefixed frames) are
|
||||
peeked here and dispatched via the `push/` package. The `push.Registry` lets
|
||||
callers register handlers for specific notification names;
|
||||
`maintnotifications/push_notification_handler.go` is how `maintnotifications`
|
||||
plugs in.
|
||||
|
||||
### Maintenance notifications (`maintnotifications/`)
|
||||
|
||||
This is a non-trivial subsystem worth understanding before touching
|
||||
cluster/handoff code. It listens for RESP3 push notifications about cluster
|
||||
maintenance (`MOVING`, `MIGRATING`, `MIGRATED`, `FAILING_OVER`, `FAILED_OVER`
|
||||
for standalone; `SMIGRATING`, `SMIGRATED` for cluster) and performs seamless
|
||||
connection handoff to new endpoints. Key pieces:
|
||||
|
||||
- `manager.go` — coordinates state transitions.
|
||||
- `handoff_worker.go` — moves in-flight ops to new connections.
|
||||
- `pool_hook.go` — integrates with `internal/pool` to mark/replace connections.
|
||||
- `circuit_breaker.go` — backs off when the upstream is unhealthy.
|
||||
- `state.go` — per-connection state machine.
|
||||
- E2E coverage lives in `maintnotifications/e2e/` and drives a fault-injector /
|
||||
RESP proxy (`cae-resp-proxy`).
|
||||
|
||||
Configuration is via `redis.Options.MaintNotificationsConfig`; modes are
|
||||
`ModeAuto` (default), `ModeEnabled` (require server support), `ModeDisabled`.
|
||||
RESP3 (`Protocol: 3`) is required.
|
||||
|
||||
### Authentication (`auth/`, `internal/auth/streaming`)
|
||||
|
||||
Four credential sources, in priority order: streaming provider (e.g. Entra ID
|
||||
via `go-redis-entraid`), context-based provider, function provider, static
|
||||
`Username`/`Password`. The streaming provider is what enables token rotation
|
||||
without reconnecting — the listener in `auth/reauth_credentials_listener.go`
|
||||
issues `AUTH` on each refresh.
|
||||
|
||||
### Internal helpers
|
||||
|
||||
- `internal/hscan` — struct scanning for `HGETALL` results (`Scan` interface
|
||||
re-exported as `redis.Scanner`).
|
||||
- `internal/hashtag` — extracts `{tag}` segments for cluster slot routing.
|
||||
- `internal/routing` — aggregator policies and shard pickers used by
|
||||
`ClusterClient` for multi-shard commands.
|
||||
- `internal/otel` — small OpenTelemetry shim used to keep root free of telemetry
|
||||
deps; full instrumentation lives in `extra/redisotel-native`.
|
||||
|
||||
## Architectural specs
|
||||
|
||||
Read the relevant design doc **before** changing code in that subsystem. They
|
||||
cover invariants and decisions that aren't obvious from the code, and are plain
|
||||
markdown any tool or editor can open:
|
||||
|
||||
- `.claude/specs/pool.md` — connection pool: `wantConn` queue and FIFO
|
||||
discipline, `ConnState` machine, dial retry/backoff, hook integration, the
|
||||
re-auth/handoff coexistence contract.
|
||||
- `.claude/specs/cluster-routing.md` — slot computation, MOVED/ASK redirection,
|
||||
request/response policies, aggregators, replica routing, topology reload,
|
||||
cross-slot rules.
|
||||
- `.claude/specs/maintnotifications.md` — RESP3 push notification protocol, mode
|
||||
handshake, per-conn state, handoff worker pool, circuit breaker, endpoint-type
|
||||
resolution, cluster vs. standalone differences.
|
||||
|
||||
## Conventions
|
||||
|
||||
- New `Cmder` type → also implement `SetVal` (the custom vet `setval` check
|
||||
enforces this; `SetErr` is on the embedded `baseCmd`).
|
||||
- Wrap errors with custom error types that implement `Unwrap`, or use
|
||||
`fmt.Errorf("...: %w", err)`. Always call `cmd.SetErr(...)` after wrapping so
|
||||
typed-error checks still pass.
|
||||
- `gofumpt` + `goimports -local github.com/redis/go-redis` is the formatter
|
||||
(`make fmt`); CI runs both.
|
||||
- Don't log directly — use `internal.Logger` (set via `redis.SetLogger`);
|
||||
`logging.Disable()` is called in tests.
|
||||
- Version-gate Redis-version-specific tests with `SkipBeforeRedisVersion` /
|
||||
`SkipAfterRedisVersion` rather than skipping at the suite level.
|
||||
|
||||
### Commits and PRs
|
||||
|
||||
Conventional Commits, short and exact — `<type>(<scope>): <imperative summary>`.
|
||||
Subject ≤50 chars (hard cap 72), imperative ("add", not "added"), no trailing
|
||||
period. Body only when the *why* isn't obvious from the diff; wrap at 72.
|
||||
|
||||
- Types: `feat`, `fix`, `refactor`, `perf`, `docs`, `test`, `chore` (also
|
||||
`build`, `ci`, `style`, `revert`).
|
||||
- Scope = the subsystem touched, lowercase: `pool`, `conn`, `pubsub`,
|
||||
`sentinel`, `retry`, `command`/`cmd`, `vectorset`, `otel`, `streams`, `push`,
|
||||
`deps`, `ci`, `tests`, `docs`. Omit only for genuinely cross-cutting changes.
|
||||
- Breaking change: `feat(scope)!: ...` plus a `BREAKING CHANGE:` body line.
|
||||
Reference issues/PRs at the end — `Closes #42`, `Refs #17`.
|
||||
- **No AI-attribution trailer.** Do not add `Co-Authored-By: …`, "Generated with
|
||||
…", or any AI-attribution line to commits or PR bodies in this repo.
|
||||
|
||||
## Repo-specific tooling
|
||||
|
||||
`.claude/` holds shared AI config:
|
||||
|
||||
- `commands/` — slash commands (e.g. `check-ci`, which summarizes a PR's CI).
|
||||
- `skills/` — task playbooks: `testing`, `add-command`, `commit-style`,
|
||||
`update-ci-image`, `prepare-release`.
|
||||
- `specs/` — the architecture docs listed above.
|
||||
|
||||
For Claude Code, the skills auto-trigger from their descriptions. For other
|
||||
tools, each `SKILL.md` is plain markdown you can open and follow directly.
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
Repository guidance — layout, commands, architecture, and conventions — is
|
||||
maintained in [`AGENTS.md`](AGENTS.md) so it stays shared across AI tools. It is
|
||||
imported below; edit `AGENTS.md`, not this file, for that content.
|
||||
|
||||
@AGENTS.md
|
||||
|
||||
## Claude-specific
|
||||
|
||||
- Repo-local **skills** under `.claude/skills/` auto-trigger from their
|
||||
descriptions — no need to invoke them manually (the set is listed in
|
||||
`AGENTS.md`).
|
||||
- **Slash commands** under `.claude/commands/` (e.g. `/check-ci`) are available
|
||||
in-session.
|
||||
- Architectural **specs** under `.claude/specs/` are read on demand; open the
|
||||
relevant one before changing that subsystem.
|
||||
+1
-1
@@ -37,7 +37,7 @@ Here's how to get started with your code contribution:
|
||||
> Note: this clones and builds the docker containers specified in `docker-compose.yml`, to understand more about
|
||||
> the infrastructure that will be started you can check the `docker-compose.yml`. You also have the possiblity
|
||||
> to specify the redis image that will be pulled with the env variable `CLIENT_LIBS_TEST_IMAGE`.
|
||||
> By default the docker image that will be pulled and started is `redislabs/client-libs-test:8.2.1-pre`.
|
||||
> By default the docker image that will be pulled and started is `redislabs/client-libs-test:8.10.0`.
|
||||
> If you want to test with newer Redis version, using a newer version of `redislabs/client-libs-test` should work out of the box.
|
||||
|
||||
4. While developing, make sure the tests pass by running `make test` (if you have the docker containers running, `make test.ci` may be sufficient).
|
||||
|
||||
+32
-3
@@ -1,8 +1,8 @@
|
||||
GO_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | sort)
|
||||
REDIS_VERSION ?= 8.8
|
||||
REDIS_VERSION ?= 8.10
|
||||
RE_CLUSTER ?= false
|
||||
RCE_DOCKER ?= true
|
||||
CLIENT_LIBS_TEST_IMAGE ?= redislabs/client-libs-test:8.8.0
|
||||
CLIENT_LIBS_TEST_IMAGE ?= redislabs/client-libs-test:8.10.0
|
||||
|
||||
docker.start:
|
||||
export RE_CLUSTER=$(RE_CLUSTER) && \
|
||||
@@ -71,6 +71,35 @@ test.ci.skip-vectorsets:
|
||||
cd internal/customvet && go build .
|
||||
go vet -vettool ./internal/customvet/customvet
|
||||
|
||||
# Replay the parametrized command integration suites through the AutoPipeliner
|
||||
# Cmdable faces (blocking + async) to prove the commands behave the same
|
||||
# batched as on a plain client. Selected via GOREDIS_TEST_SUBJECT (see
|
||||
# newUniversalSubject in main_test.go). Requires the docker env
|
||||
# (make docker.start).
|
||||
#
|
||||
# The focus covers every parametrized suite (some via the Commands/BitCount
|
||||
# substrings — Describe names differ in case). Skipped on purpose: DDL Commands (needs the cluster topology the
|
||||
# lightweight BeforeSuite doesn't build), HotKeys Commands (same), and
|
||||
# AutoPipeline Blocking Commands (the AP suite itself — running it through an
|
||||
# AP subject would nest engines).
|
||||
test.autopipeline-subjects:
|
||||
# RE_CLUSTER=true forces the lightweight BeforeSuite (no sentinel/ring/cluster
|
||||
# setup): the command suite only needs the standalone Redis, and running the
|
||||
# full stateful BeforeSuite twice (once per subject) against the same server
|
||||
# corrupts replication state and fails the second run. Both faces then run
|
||||
# cleanly against the same env. The explicit -timeout keeps a hang from
|
||||
# eating go test's 10m default per subject.
|
||||
set -e; for subj in ap-blocking ap-async; do \
|
||||
echo "=== command suite via GOREDIS_TEST_SUBJECT=$$subj ==="; \
|
||||
(export RE_CLUSTER=true && \
|
||||
export RCE_DOCKER=$(RCE_DOCKER) && \
|
||||
export REDIS_VERSION=$(REDIS_VERSION) && \
|
||||
export GOREDIS_TEST_SUBJECT=$$subj && \
|
||||
go test -v . -race -skip Example -run TestGinkgoSuite -timeout 8m \
|
||||
-ginkgo.focus='Commands|RediSearch commands|Probabilistic commands|RedisTimeseries commands|Redis VectorSet commands|BitCount|ScanIterator|Advanced JSON' \
|
||||
-ginkgo.skip='AutoPipeline Blocking Commands|DDL Commands|HotKeys Commands'); \
|
||||
done
|
||||
|
||||
bench:
|
||||
export RE_CLUSTER=$(RE_CLUSTER) && \
|
||||
export RCE_DOCKER=$(RCE_DOCKER) && \
|
||||
@@ -101,7 +130,7 @@ test.e2e.logic:
|
||||
go test -v -run "TestCreateTestFaultInjectorLogic|TestFaultInjectorClientCreation" ./maintnotifications/e2e/
|
||||
@echo "Logic tests completed!"
|
||||
|
||||
.PHONY: all test test.ci test.ci.skip-vectorsets bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop
|
||||
.PHONY: all test test.ci test.ci.skip-vectorsets test.autopipeline-subjects bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop
|
||||
|
||||
build:
|
||||
export RE_CLUSTER=$(RE_CLUSTER) && \
|
||||
|
||||
+179
@@ -21,6 +21,7 @@ In `go-redis` we are aiming to support the last three releases of Redis. Current
|
||||
- [Redis 8.2](https://raw.githubusercontent.com/redis/redis/8.2/00-RELEASENOTES) - using Redis CE 8.2
|
||||
- [Redis 8.4](https://raw.githubusercontent.com/redis/redis/8.4/00-RELEASENOTES) - using Redis CE 8.4
|
||||
- [Redis 8.8](https://raw.githubusercontent.com/redis/redis/8.8/00-RELEASENOTES) - using Redis CE 8.8
|
||||
- [Redis 8.10](https://raw.githubusercontent.com/redis/redis/8.10/00-RELEASENOTES) - using Redis CE 8.10
|
||||
|
||||
Although the `go.mod` states it requires at minimum `go 1.24`, our CI is configured to run the tests against all supported
|
||||
versions of Redis and multiple versions of Go ([1.24](https://go.dev/doc/devel/release#go1.24.0), oldstable, and stable). We observe that some modules related test may not pass with
|
||||
@@ -78,9 +79,13 @@ surface. The API is experimental and may change in a future release.
|
||||
- [StreamingCredentialsProvider (e.g. entra id, oauth)](#1-streaming-credentials-provider-highest-priority) (experimental)
|
||||
- [Pub/Sub](https://redis.uptrace.dev/guide/go-redis-pubsub.html).
|
||||
- [Pipelines and transactions](https://redis.uptrace.dev/guide/go-redis-pipelines.html).
|
||||
- [Automatic pipelining](#automatic-pipelining) (experimental) — batches concurrent
|
||||
commands into pipelines for you; meant for high-throughput / high-load / scale
|
||||
use cases.
|
||||
- [Scripting](https://redis.uptrace.dev/guide/lua-scripting.html).
|
||||
- [Redis Sentinel](https://redis.uptrace.dev/guide/go-redis-sentinel.html).
|
||||
- [Redis Cluster](https://redis.uptrace.dev/guide/go-redis-cluster.html).
|
||||
- [Client-side caching](#client-side-caching).
|
||||
- [Redis Performance Monitoring](https://redis.uptrace.dev/guide/redis-performance-monitoring.html).
|
||||
- [Redis Probabilistic [RedisStack]](https://redis.io/docs/data-types/probabilistic/)
|
||||
- [Customizable read and write buffers size.](#custom-buffer-sizes)
|
||||
@@ -285,6 +290,50 @@ rdb := redis.NewClient(&redis.Options{
|
||||
})
|
||||
```
|
||||
|
||||
### Client-side caching
|
||||
|
||||
go-redis supports server-assisted client-side caching for standalone clients.
|
||||
Eligible read replies are stored in the application's memory, so repeated reads
|
||||
can avoid a Redis round trip. Redis tracks which keys each connection has read
|
||||
and sends RESP3 invalidation notifications when those keys change. go-redis
|
||||
uses those notifications to evict affected entries automatically.
|
||||
|
||||
> **Experimental:** The client-side caching API may change in a minor release.
|
||||
|
||||
Enable the built-in bounded cache with `ClientSideCacheConfig`:
|
||||
|
||||
```go
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: "localhost:6379",
|
||||
Protocol: 3,
|
||||
DB: 0,
|
||||
ClientSideCacheConfig: &redis.ClientSideCacheConfig{
|
||||
MaxEntries: 10_000,
|
||||
},
|
||||
})
|
||||
defer rdb.Close()
|
||||
```
|
||||
|
||||
Client-side caching currently requires RESP3, a standalone client, and database
|
||||
0. Fixed `Username` and `Password` values are supported. It is disabled when a
|
||||
dynamic credential provider is configured, because cached data must never be
|
||||
reused after the client's ACL identity changes. Only deterministic read
|
||||
commands supported by the cache are stored; writes and streaming responses
|
||||
bypass it.
|
||||
|
||||
While client-side caching is enabled, go-redis rejects `SELECT`, `AUTH`,
|
||||
`HELLO` with arguments, `RESET`, `CLIENT TRACKING`, and raw `SUBSCRIBE`,
|
||||
`PSUBSCRIBE`, or `SSUBSCRIBE` commands because they would change connection
|
||||
state that the cache relies on. A guarded command also fails its whole
|
||||
pipeline. The typed `Subscribe`, `PSubscribe`, and `SSubscribe` APIs remain
|
||||
supported because they use dedicated connections.
|
||||
|
||||
Invalidations are processed asynchronously. `DrainInterval` controls how often
|
||||
idle connections are checked for them, while `MaxStaleness` can provide an
|
||||
optional upper bound on an entry's lifetime. See the
|
||||
[client-side caching example](./example/client-side-caching) for a working
|
||||
demonstration.
|
||||
|
||||
### Connecting via a redis url
|
||||
|
||||
go-redis also supports connecting via the
|
||||
@@ -340,6 +389,118 @@ rdb := redis.NewClient(&redis.Options{
|
||||
})
|
||||
```
|
||||
|
||||
### Automatic pipelining
|
||||
|
||||
**Experimental** — the API may still change. Reach for autopipelining in
|
||||
high-throughput / high-load / scale scenarios; at low concurrency a plain
|
||||
client is simpler and just as fast. A runnable usage tour and throughput
|
||||
comparison live in [`example/autopipeline`](example/autopipeline).
|
||||
|
||||
> **EXPERIMENTAL:** the autopipelining API is subject to change in a future
|
||||
> release as we gather feedback — pin your go-redis version if you adopt it.
|
||||
|
||||
When many goroutines issue commands concurrently, autopipelining batches them
|
||||
into Redis pipelines automatically — without you writing any pipeline code. It
|
||||
comes in two faces:
|
||||
|
||||
- **`AutoPipeline()` — blocking, drop-in.** Each command call blocks until it
|
||||
executes and returns its own value/error, exactly like a normal client, so
|
||||
existing code keeps working unchanged. Under concurrency the engine coalesces
|
||||
commands from all goroutines into deep, back-to-back pipelines (a single
|
||||
ordered batch stream by default), reaching several times a plain client's
|
||||
executed commands per second in the same environment — roughly an order of
|
||||
magnitude with a parallel-batch config (`MaxConcurrentBatches` > 1 with
|
||||
`Unordered`). Per-goroutine ordering is preserved.
|
||||
- **`AsyncAutoPipeline()` — deferred, highest throughput.** Command calls return
|
||||
immediately; you submit a window of commands and read their results afterward,
|
||||
which keeps each pipeline deep — tens of times a plain client's throughput.
|
||||
Ordered by default. Absolute numbers depend heavily on the machine, network
|
||||
path and server; see `autopipeline_bench_README.md` for the benchmark
|
||||
methodology and multipliers.
|
||||
|
||||
```go
|
||||
rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379"})
|
||||
defer rdb.Close()
|
||||
ctx := context.Background()
|
||||
|
||||
// Blocking face: drop-in for a normal client, batched under the hood.
|
||||
ap, err := rdb.AutoPipeline()
|
||||
if err != nil { // invalid AutoPipelineOptions, or the client is closed
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer ap.Close()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 1000; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
key := fmt.Sprintf("key:%d", i)
|
||||
if err := ap.Set(ctx, key, i, 0).Err(); err != nil { // blocks until executed
|
||||
log.Printf("set %s: %v", key, err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
```
|
||||
|
||||
For maximum throughput, submit a window on the async face and read later:
|
||||
|
||||
```go
|
||||
ctx := context.Background()
|
||||
ap, err := rdb.AsyncAutoPipeline() // ordered by default
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer ap.Close()
|
||||
|
||||
cmds := make([]*redis.StatusCmd, 0, 200)
|
||||
for i := 0; i < 200; i++ {
|
||||
cmds = append(cmds, ap.Set(ctx, fmt.Sprintf("key:%d", i), i, 0)) // returns immediately
|
||||
}
|
||||
for _, cmd := range cmds {
|
||||
if err := cmd.Err(); err != nil { // blocks until executed
|
||||
log.Printf("set: %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Each face has a no-argument form that uses `Options.AutoPipelineOptions` (or the
|
||||
built-in default) and a `WithOptions` form that takes an explicit
|
||||
`*AutoPipelineOptions`; both return `(*AutoPipeliner, error)` — the error is
|
||||
non-nil for an invalid config or a closed client (e.g.
|
||||
`ap, err := rdb.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{MaxConcurrentBatches: 8, Unordered: true})`);
|
||||
a handful of parallel batches saturates the link — more permits only add
|
||||
overlapping batches without deepening them.
|
||||
They work on `ClusterClient` too: commands are routed to the correct shard per
|
||||
key, so a single batch may span many slots; ordering across nodes is per key
|
||||
(same-key commands stay in order, different nodes' sub-pipelines run
|
||||
concurrently). Because batches share a few pipeline connections, autopipelining
|
||||
also needs far fewer connections than a plain client at the same concurrency
|
||||
(see `PipelinePoolSize`). Autopipelining is only a win under concurrency (or
|
||||
windowed submission) — a single goroutine issuing one blocking command at a
|
||||
time sees little benefit, and a hand-written `Pipeline()` is still fastest when
|
||||
you can batch by hand.
|
||||
|
||||
Caveats: a command's context is not honored once it is queued (batches execute
|
||||
on the autopipeliner's own context) — use a plain client for per-command
|
||||
deadlines. Blocking commands (`BLPOP`, `WAIT`, ...) are never batched and run
|
||||
directly on your context — as are `SHUTDOWN` and `MONITOR`, which would
|
||||
poison a shared pipeline connection — and `Do` also bypasses batching with plain
|
||||
`Client.Do` semantics — prefer the typed methods (`ap.Set`, `ap.Get`, ...). On
|
||||
a dropped connection a batch is retried whole (up to `MaxRetries`), so
|
||||
non-idempotent commands may execute twice. Both faces return a cached,
|
||||
client-shared instance: the first call's config wins and `Close` stops it for
|
||||
all callers. Hooks may read command results (the engine hands a hook running
|
||||
on the dispatch goroutine the same view a plain pipeline hook gets), but a
|
||||
hook must never issue a command on the same autopipeliner and wait for it —
|
||||
the nested command needs the very dispatch slot the hook is holding, and the
|
||||
engine only recovers by failing that flush after its 30s permit backstops.
|
||||
`Options.Limiter` is consulted once per batch dispatch (as with a manual
|
||||
pipeline), not once per command. An autopipeliner created on a
|
||||
`WithTimeout`/`WithReadTimeout` clone is not stopped by the parent's `Close` —
|
||||
close it explicitly.
|
||||
|
||||
### Advanced Configuration
|
||||
|
||||
go-redis supports extending the client identification phase to allow projects to send their own custom client identification.
|
||||
@@ -457,6 +618,24 @@ vals, err := rdb.Eval(ctx, "return {KEYS[1],ARGV[1]}", []string{"key"}, "hello")
|
||||
res, err := rdb.Do(ctx, "set", "key", "value").Result()
|
||||
```
|
||||
|
||||
### Raw commands and connection state
|
||||
|
||||
`Do` sends the command verbatim on whichever pooled connection happens to be
|
||||
free. For keyspace commands that is all you need. It is the wrong tool for
|
||||
any command that alters **connection session state** — `SELECT`,
|
||||
`CLIENT SETNAME`, `CLIENT TRACKING`, `RESET`, `HIMPORT PREPARE`/`DISCARD`,
|
||||
and similar: the state lands on (or is wiped from) a single arbitrary
|
||||
connection, later commands are served by other connections that don't share
|
||||
it, and the affected connection eventually returns to the pool and serves
|
||||
unrelated callers. The result is nondeterministic behavior that typed APIs
|
||||
manage for you — for example, the typed `HImport*` methods keep a
|
||||
client-side registry and replay fieldsets onto every connection that needs
|
||||
them, while a raw `Do(ctx, "himport", "prepare", ...)` bypasses that
|
||||
entirely, with no replay, recovery, or discard propagation.
|
||||
|
||||
For session-scoped work without a typed API, hold a dedicated connection
|
||||
(`client.Conn()`) for its whole lifetime and close it afterwards.
|
||||
|
||||
## Typed Errors
|
||||
|
||||
go-redis provides typed error checking functions for common Redis errors:
|
||||
|
||||
+239
@@ -1,5 +1,244 @@
|
||||
# Release Notes
|
||||
|
||||
# 9.22.0 (2026-08-03)
|
||||
|
||||
This is a minor release introducing two flagship (experimental) features — **client-side caching** and **automatic pipelining** — alongside support for Redis 8.10, new commands, and a large batch of stability and parser-robustness fixes. It consolidates everything shipped in 9.22.0-beta.1, so the notes below cover the full 9.21.0 → 9.22.0 upgrade.
|
||||
|
||||
⚠️ Two changes to be aware of when upgrading from 9.21.0:
|
||||
|
||||
- **Default configuration values changed** ([#3918](https://github.com/redis/go-redis/pull/3918)): read/write timeouts, retry backoff, cluster state reload interval, and TCP keep-alive defaults are now aligned with the cross-SDK configuration proposal (see the highlight below). Explicitly configured values are unaffected.
|
||||
- **`WaitAOF` return type corrected** ([#3888](https://github.com/redis/go-redis/pull/3888)): `WaitAOF` now returns `*IntSliceCmd`, matching the two-integer reply of `WAITAOF` (previously `*IntCmd`, which failed to parse the reply at runtime). Code referencing the old return type needs a one-line update.
|
||||
|
||||
## 🚀 Highlights
|
||||
|
||||
### Client-Side Caching (Experimental)
|
||||
|
||||
The standalone `Client` gains server-assisted client-side caching built on RESP3 `CLIENT TRACKING`. Enable it by setting `ClientSideCacheConfig` in `Options` (or supply your own cache via `ClientSideCache` — e.g. to share one cache across clients). Cacheable read results are served from a local in-process cache and invalidated automatically when the server reports a change, cutting round trips for read-heavy workloads.
|
||||
|
||||
The invalidation architecture is selected by `ClientSideCacheStrategy`; the default (and currently only) strategy is `CSCStrategySharedTracking`: one shared cache, every pool connection runs plain `CLIENT TRACKING ON`, and a background drainer applies buffered invalidations — portable (no BCAST) and consistent with the other Redis client libraries. Requirements and guardrails: RESP3 (`Protocol: 3`), standalone client, DB 0 only; commands that would change the connection identity (`SELECT`, `AUTH`, ...) are rejected while caching is enabled, and CSC is disabled when a credentials provider is set (fixed `Username`/`Password` work and are namespaced). See the README's [client-side caching section](README.md#client-side-caching) and the runnable [example](example/client-side-caching).
|
||||
|
||||
**Experimental:** the API may change in a minor release.
|
||||
|
||||
([#3941](https://github.com/redis/go-redis/pull/3941)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
|
||||
### Automatic Pipelining (Experimental)
|
||||
|
||||
`AutoPipeliner` is a background batcher that coalesces commands from many concurrent goroutines into Redis pipelines, multiplying throughput without any manual pipeline management. It comes in two faces, available on `Client` and `ClusterClient` (and configurable via `Options.AutoPipelineOptions` / `UniversalOptions.AutoPipelineOptions`):
|
||||
|
||||
- **`AutoPipeline()`** — the blocking face: a drop-in `Cmdable` where each call blocks until executed, exactly like a plain client, while concurrent callers' commands batch together under the hood (measured locally over loopback: ~1M+ SET/sec vs ~100k unpipelined; indicative, not a guarantee). Per-goroutine command order is preserved.
|
||||
- **`AsyncAutoPipeline()`** — the deferred face: command calls return immediately and every typed result accessor (`Val`/`Result`/`Err`/...) blocks until the command has executed. Submit a window of commands, then read the results, to keep pipelines deep (~2–3M SET/sec locally; indicative).
|
||||
|
||||
`AutoPipelineOptions` controls batching: `MaxBatchSize` (soft target, default 200; the blocking face's preset uses 300), `MaxBatchBytes` (approximate payload cap so huge values flush as several bounded writes), `MaxFlushDelay` with optional `AdaptiveDelay` (delay scales down as the queue fills), and `MaxConcurrentBatches` (default 1 = a single ordered batch stream; raising it requires `Unordered: true`, so ordering is never lost by accident — `Validate()` rejects the combination otherwise). A usage tour and throughput comparison live in [`example/autopipeline`](example/autopipeline).
|
||||
|
||||
**Experimental:** the API may change in a future release — pin your go-redis version if you adopt it.
|
||||
|
||||
([#3942](https://github.com/redis/go-redis/pull/3942)) by [@ndyakov](https://github.com/ndyakov), with help from [@cxljs](https://github.com/cxljs)
|
||||
|
||||
### Redis 8.10 Support
|
||||
|
||||
This release adds support for **Redis 8.10**. The README's supported-versions list now includes Redis 8.10, and CI runs the full suite against the `redislabs/client-libs-test:8.10.0` image by default ([#3920](https://github.com/redis/go-redis/pull/3920), [#3940](https://github.com/redis/go-redis/pull/3940)).
|
||||
|
||||
Coverage for the new commands and options that ship with Redis 8.10:
|
||||
|
||||
- **`HIMPORT`** ([#3919](https://github.com/redis/go-redis/pull/3919)) — bulk hash import via server-side fieldsets, exposed as `HImportPrepare`, `HImportSet`, `HImportDiscard`, and `HImportDiscardAll`. Fieldsets are session state scoped to a single physical connection, which does not mix well with connection pooling — so the client keeps a versioned fieldset registry and lazily replays the `PREPARE` on whichever pooled connection executes a `SET` that needs it, at most once per connection, with no extra round trip (the `PREPARE` is injected into the same write as the `SET`).
|
||||
- **`LMOVEM` / `BLMOVEM`** ([#3913](https://github.com/redis/go-redis/pull/3913)) — move multiple elements between lists in one call.
|
||||
- **`SUNIONCARD` / `SDIFFCARD`** ([#3897](https://github.com/redis/go-redis/pull/3897)) — cardinality of set union/difference without materializing the result.
|
||||
- **`XREAD` / `XREADGROUP` `MAXCOUNT` and `MAXSIZE`** ([#3898](https://github.com/redis/go-redis/pull/3898)) — bound how much data a stream read returns.
|
||||
- **`TS.READ`** ([#3896](https://github.com/redis/go-redis/pull/3896)), **`TS.QUERYLABELS`** ([#3926](https://github.com/redis/go-redis/pull/3926)), **`TS.NRANGE` / `TS.NREVRANGE`** ([#3870](https://github.com/redis/go-redis/pull/3870)) with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)), and **`EXCLUDEEMPTY`** on `TS.MRANGE` / `TS.MREVRANGE` ([#3912](https://github.com/redis/go-redis/pull/3912)) — new time-series query surface.
|
||||
- **`FT.ALIASLIST`** ([#3925](https://github.com/redis/go-redis/pull/3925)), **`COLLECT` reducer for `FT.AGGREGATE`** ([#3886](https://github.com/redis/go-redis/pull/3886)), **`RERANK` on HNSW vector fields in `FT.CREATE`** ([#3927](https://github.com/redis/go-redis/pull/3927)), and **`FT.HYBRID` timeout warnings** ([#3911](https://github.com/redis/go-redis/pull/3911)) — search coverage.
|
||||
|
||||
### Cross-SDK Aligned Defaults
|
||||
|
||||
Default configuration values now follow the cross-SDK configuration proposal shared by all Redis client libraries ([#3918](https://github.com/redis/go-redis/pull/3918)):
|
||||
|
||||
| Setting | Old default | New default |
|
||||
|---|---|---|
|
||||
| `ReadTimeout` / `WriteTimeout` | 3s | 5s |
|
||||
| Retry backoff (min/max) | 8ms / 512ms | 10ms / 1s |
|
||||
| Cluster state reload interval | 10s | 60s |
|
||||
| TCP keep-alive | 5min period | 30s idle / 5s interval / 3 probes (`net.KeepAliveConfig`) |
|
||||
|
||||
Applications that set these values explicitly are unaffected; applications relying on the old defaults inherit the new ones.
|
||||
|
||||
### Data-Race and Parser Hardening Sweep
|
||||
|
||||
A systematic audit fixed data races across the client — hooks (`AddHook`, [#3868](https://github.com/redis/go-redis/pull/3868)), `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), cluster node slices ([#3861](https://github.com/redis/go-redis/pull/3861)), pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), maintenance notifications ([#3894](https://github.com/redis/go-redis/pull/3894), [#3872](https://github.com/redis/go-redis/pull/3872)), pool handoff ([#3876](https://github.com/redis/go-redis/pull/3876)), and `redisotel` ([#3881](https://github.com/redis/go-redis/pull/3881)) — and hardened the RESP parsers against malformed or unexpected replies: over-reads on nil replies ([#3874](https://github.com/redis/go-redis/pull/3874)), integer overflow when skipping map/attribute bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), unhashable RESP3 map keys ([#3873](https://github.com/redis/go-redis/pull/3873)), odd-length flat replies ([#3900](https://github.com/redis/go-redis/pull/3900)), mismatched declared array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), unexpected extra reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), and nil elements in numeric/bool slice replies ([#3922](https://github.com/redis/go-redis/pull/3922)).
|
||||
|
||||
### PubSub `Receive` Hang Fix
|
||||
|
||||
`PeekPushNotificationName` blocked until 36 bytes were buffered, so a short subscribe confirmation (channel name of six or fewer characters) on an otherwise idle connection hung `PubSub.Receive` forever — a regression introduced in 9.20.1 by [#3842](https://github.com/redis/go-redis/pull/3842). The peek now parses whatever is already buffered and only waits for one more byte when the frame prefix is valid but incomplete. Fixes [#3935](https://github.com/redis/go-redis/issues/3935).
|
||||
|
||||
([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov)
|
||||
|
||||
### Correct Cluster Transaction Retries
|
||||
|
||||
The cluster transaction pipeline treated a `MULTI`...`EXEC` block as independently retryable commands, which could scatter a transaction across nodes or send malformed transactions on retry. Redirects (`MOVED`/`ASK`/`TRYAGAIN`) and aborts are now handled at the whole-transaction level, matching Redis transaction semantics: the transaction is re-routed and retried as a unit, never partially ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs).
|
||||
|
||||
### Credential Redaction in Command Tracing
|
||||
|
||||
`rediscmd.AppendCmd` — used by `redisotel` and `rediscensus` to render commands into span attributes — now redacts credential arguments as `<redacted>`: `AUTH`, `HELLO ... AUTH`, `CONFIG SET` of `requirepass` / `masterauth` / TLS key passphrases, `ACL SETUSER` password rules, and `MIGRATE ... AUTH`/`AUTH2`. The client sends `HELLO ... AUTH` on every handshake and `AUTH` on every streaming-credentials rotation through the regular hook chain, so tracing hooks previously captured credentials even when the application never issued an auth command itself ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e).
|
||||
|
||||
## ✨ New Features
|
||||
|
||||
- **Client-side caching**: server-assisted caching for the standalone client via `ClientSideCacheConfig` / `ClientSideCache`, with the `CSCStrategySharedTracking` invalidation strategy ([#3941](https://github.com/redis/go-redis/pull/3941)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **Automatic pipelining**: `AutoPipeline()` (blocking) and `AsyncAutoPipeline()` (deferred results) on `Client` and `ClusterClient`, configured via `AutoPipelineOptions` ([#3942](https://github.com/redis/go-redis/pull/3942)) by [@ndyakov](https://github.com/ndyakov), with help from [@cxljs](https://github.com/cxljs)
|
||||
- **`HIMPORT` command family**: `HImportPrepare` / `HImportSet` / `HImportDiscard` / `HImportDiscardAll` with lazy per-connection fieldset prepare replay ([#3919](https://github.com/redis/go-redis/pull/3919)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`LMOVEM` / `BLMOVEM`**: move multiple list elements in one call, with `COUNT` (up to N) or `EXACTLY` (all-or-nothing) semantics via `LMoveMArgs` ([#3913](https://github.com/redis/go-redis/pull/3913)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`SUnionCard` / `SDiffCard`**: cardinality of set union/difference ([#3897](https://github.com/redis/go-redis/pull/3897)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`XRead` / `XReadGroup` `MAXCOUNT` / `MAXSIZE`**: bound stream read responses by entry count or payload size ([#3898](https://github.com/redis/go-redis/pull/3898)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`TS.READ`**: read samples from a series starting at a given timestamp, with `TSReadEarliest` (`-`), `TSReadLatest` (`+`), and `TSReadNew` (`$`) sentinels ([#3896](https://github.com/redis/go-redis/pull/3896)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`TS.QUERYLABELS`**: query label names/values across time series ([#3926](https://github.com/redis/go-redis/pull/3926)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`TS.NRANGE` / `TS.NREVRANGE`**: range queries across multiple series ([#3870](https://github.com/redis/go-redis/pull/3870)) by [@ofekshenawa](https://github.com/ofekshenawa), with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`TS.MRANGE` / `TS.MREVRANGE` `EXCLUDEEMPTY`**: skip series with no samples in the result ([#3912](https://github.com/redis/go-redis/pull/3912)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`FT.ALIASLIST`**: list all index aliases ([#3925](https://github.com/redis/go-redis/pull/3925)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`FT.AGGREGATE` `COLLECT` reducer**: collect grouped values into an array ([#3886](https://github.com/redis/go-redis/pull/3886)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`FT.CREATE` `RERANK`**: `RERANK` parameter on HNSW vector field definitions ([#3927](https://github.com/redis/go-redis/pull/3927)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`FT.HYBRID` timeout warnings**: timeout warnings are now populated in hybrid search results ([#3911](https://github.com/redis/go-redis/pull/3911)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`FT.HYBRID` KNN `SHARD_K_RATIO`** (Redis 8.8+): per-shard K ratio for KNN clauses ([#3841](https://github.com/redis/go-redis/pull/3841)) by [@ndyakov](https://github.com/ndyakov)
|
||||
|
||||
## 🐛 Bug Fixes
|
||||
|
||||
- **PubSub `Receive` hang**: peek push-notification names without demanding 36 buffered bytes, fixing a hang on short subscribe confirmations (fixes [#3935](https://github.com/redis/go-redis/issues/3935), regression from 9.20.1) ([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **Cluster transactions**: re-route the whole tx pipeline on redirect/abort instead of per-command ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs)
|
||||
- **Credential leak in traces**: `rediscmd.AppendCmd` redacts credential arguments (`AUTH`, `HELLO ... AUTH`, `CONFIG SET` secret params, `ACL SETUSER` password rules, `MIGRATE AUTH`/`AUTH2`), so `redisotel` / `rediscensus` span attributes no longer contain passwords ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e)
|
||||
- **`WaitAOF` return type**: returns `*IntSliceCmd` matching the two-integer `WAITAOF` reply ([#3888](https://github.com/redis/go-redis/pull/3888)) by [@CipherN9](https://github.com/CipherN9)
|
||||
- **`Ring.Publish` routing**: publish to the shard that owns the topic instead of a round-robined one ([#3893](https://github.com/redis/go-redis/pull/3893)) by [@dkindel](https://github.com/dkindel)
|
||||
- **Pool `OnRemove` hooks**: fire `OnRemove` on `putConn` eviction paths so removal hooks see every evicted connection ([#3932](https://github.com/redis/go-redis/pull/3932)) by [@cxljs](https://github.com/cxljs)
|
||||
- **`UniversalClient` `InfoMap`**: added `InfoMap` to the `Cmdable` interface ([#3904](https://github.com/redis/go-redis/pull/3904)) by [@nazarli-shabnam](https://github.com/nazarli-shabnam)
|
||||
- **`SlowLogGet` context**: pass the caller's context instead of a background one ([#3915](https://github.com/redis/go-redis/pull/3915)) by [@sonnemusk](https://github.com/sonnemusk)
|
||||
- **`ModuleLoadex` nil config**: return an error instead of panicking on nil config ([#3916](https://github.com/redis/go-redis/pull/3916)) by [@sonnemusk](https://github.com/sonnemusk)
|
||||
- **`ParseURL` IPv6 hosts**: keep single brackets for IPv6 hosts without a port ([#3882](https://github.com/redis/go-redis/pull/3882)) by [@sueun-dev](https://github.com/sueun-dev)
|
||||
- **`ParseURL` durations**: treat unit durations `<= 0` as disabled ([#3866](https://github.com/redis/go-redis/pull/3866)) by [@sueun-dev](https://github.com/sueun-dev)
|
||||
- **Nil `*uint8` encoding**: encode nil `*uint8` as `"0"` like other numeric pointers ([#3869](https://github.com/redis/go-redis/pull/3869)) by [@sueun-dev](https://github.com/sueun-dev)
|
||||
- **`JSONSliceCmd` read errors**: return the read error from `readReply` instead of swallowing it ([#3903](https://github.com/redis/go-redis/pull/3903)) by [@saddamr3e](https://github.com/saddamr3e)
|
||||
- **RESP parser hardening**: reconcile declared entry-array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), handle nil elements in int/uint/bool slice parsers ([#3922](https://github.com/redis/go-redis/pull/3922)), drain unexpected reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), reject odd-length flat replies in Z/KeyValue parsers ([#3900](https://github.com/redis/go-redis/pull/3900)), avoid int overflow when skipping map/attr bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), don't over-read nil replies in `Reader.Discard` ([#3874](https://github.com/redis/go-redis/pull/3874)) by [@saddamr3e](https://github.com/saddamr3e); reject unhashable keys in RESP3 map parsing ([#3873](https://github.com/redis/go-redis/pull/3873)) by [@iabdullah215](https://github.com/iabdullah215)
|
||||
- **Data races**: hook state during `AddHook` ([#3868](https://github.com/redis/go-redis/pull/3868)), `onNewNode` during `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), shared masters/slaves slices in cluster ([#3861](https://github.com/redis/go-redis/pull/3861)), shared `opt.Addr` during pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), `clusterStateReloadCallback` in maintnotifications ([#3894](https://github.com/redis/go-redis/pull/3894)), conn reader in `isHealthyConn` during handoff ([#3876](https://github.com/redis/go-redis/pull/3876)) by [@saddamr3e](https://github.com/saddamr3e); handoff race window in maintnotifications ([#3872](https://github.com/redis/go-redis/pull/3872)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`redisotel`**: use `ObservableCounter` for cumulative pool stats ([#3914](https://github.com/redis/go-redis/pull/3914)) by [@Solaris-star](https://github.com/Solaris-star); avoid a data race on shared attributes during `MinIdleConns` warmup ([#3881](https://github.com/redis/go-redis/pull/3881)) by [@ndyakov](https://github.com/ndyakov)
|
||||
|
||||
## 🧰 Maintenance
|
||||
|
||||
- **Cross-SDK default alignment**: new defaults for timeouts, retry backoff, cluster state reload, and TCP keep-alive ([#3918](https://github.com/redis/go-redis/pull/3918)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **CI on Redis 8.10**: 8.10 made the default test version ([#3920](https://github.com/redis/go-redis/pull/3920)) with version gating by major.minor ([#3908](https://github.com/redis/go-redis/pull/3908)) by [@ofekshenawa](https://github.com/ofekshenawa); the test stack now runs the GA `redislabs/client-libs-test:8.10.0` image and 8.8 was dropped from the CI matrix ([#3940](https://github.com/redis/go-redis/pull/3940))
|
||||
- **Type-safe atomics**: use typed `sync/atomic` value types ([#3860](https://github.com/redis/go-redis/pull/3860)) and remove the dead `assertUnstableCommand` RESP3 path ([#3928](https://github.com/redis/go-redis/pull/3928)) by [@cxljs](https://github.com/cxljs)
|
||||
- **Docs**: clarify that `ExpireTime` / `PExpireTime` return Unix timestamps ([#3917](https://github.com/redis/go-redis/pull/3917)) by [@sonnemusk](https://github.com/sonnemusk); remove a duplicate example step ([#3875](https://github.com/redis/go-redis/pull/3875)) by [@andy-stark-redis](https://github.com/andy-stark-redis)
|
||||
|
||||
## 👥 Contributors
|
||||
|
||||
We'd like to thank all the contributors who worked on this release!
|
||||
|
||||
[@andy-stark-redis](https://github.com/andy-stark-redis), [@CipherN9](https://github.com/CipherN9), [@cxljs](https://github.com/cxljs), [@dkindel](https://github.com/dkindel), [@iabdullah215](https://github.com/iabdullah215), [@nazarli-shabnam](https://github.com/nazarli-shabnam), [@ndyakov](https://github.com/ndyakov), [@ofekshenawa](https://github.com/ofekshenawa), [@saddamr3e](https://github.com/saddamr3e), [@Solaris-star](https://github.com/Solaris-star), [@sonnemusk](https://github.com/sonnemusk), [@sueun-dev](https://github.com/sueun-dev)
|
||||
|
||||
---
|
||||
|
||||
**Full Changelog**: https://github.com/redis/go-redis/compare/v9.21.0...v9.22.0
|
||||
|
||||
# 9.22.0-beta.1 (2026-07-29)
|
||||
|
||||
This is a **beta** release adding support for Redis 8.10, new commands, and a large batch of stability and parser-robustness fixes. The 9.22.0 GA release will follow once client-side caching and auto-pipelining are merged.
|
||||
|
||||
⚠️ Two changes to be aware of when upgrading from 9.21.0:
|
||||
|
||||
- **Default configuration values changed** ([#3918](https://github.com/redis/go-redis/pull/3918)): read/write timeouts, retry backoff, cluster state reload interval, and TCP keep-alive defaults are now aligned with the cross-SDK configuration proposal (see the highlight below). Explicitly configured values are unaffected.
|
||||
- **`WaitAOF` return type corrected** ([#3888](https://github.com/redis/go-redis/pull/3888)): `WaitAOF` now returns `*IntSliceCmd`, matching the two-integer reply of `WAITAOF` (previously `*IntCmd`, which failed to parse the reply at runtime). Code referencing the old return type needs a one-line update.
|
||||
|
||||
## 🚀 Highlights
|
||||
|
||||
### Redis 8.10 Support
|
||||
|
||||
This release adds support for **Redis 8.10**. The README's supported-versions list now includes Redis 8.10, and CI runs the full suite against the `redislabs/client-libs-test:8.10.0` image by default ([#3920](https://github.com/redis/go-redis/pull/3920), [#3940](https://github.com/redis/go-redis/pull/3940)).
|
||||
|
||||
Coverage for the new commands and options that ship with Redis 8.10:
|
||||
|
||||
- **`HIMPORT`** ([#3919](https://github.com/redis/go-redis/pull/3919)) — bulk hash import via server-side fieldsets, exposed as `HImportPrepare`, `HImportSet`, `HImportDiscard`, and `HImportDiscardAll`. Fieldsets are session state scoped to a single physical connection, which does not mix well with connection pooling — so the client keeps a versioned fieldset registry and lazily replays the `PREPARE` on whichever pooled connection executes a `SET` that needs it, at most once per connection, with no extra round trip (the `PREPARE` is injected into the same write as the `SET`).
|
||||
- **`LMOVEM` / `BLMOVEM`** ([#3913](https://github.com/redis/go-redis/pull/3913)) — move multiple elements between lists in one call.
|
||||
- **`SUNIONCARD` / `SDIFFCARD`** ([#3897](https://github.com/redis/go-redis/pull/3897)) — cardinality of set union/difference without materializing the result.
|
||||
- **`XREAD` / `XREADGROUP` `MAXCOUNT` and `MAXSIZE`** ([#3898](https://github.com/redis/go-redis/pull/3898)) — bound how much data a stream read returns.
|
||||
- **`TS.READ`** ([#3896](https://github.com/redis/go-redis/pull/3896)), **`TS.QUERYLABELS`** ([#3926](https://github.com/redis/go-redis/pull/3926)), **`TS.NRANGE` / `TS.NREVRANGE`** ([#3870](https://github.com/redis/go-redis/pull/3870)) with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)), and **`EXCLUDEEMPTY`** on `TS.MRANGE` / `TS.MREVRANGE` ([#3912](https://github.com/redis/go-redis/pull/3912)) — new time-series query surface.
|
||||
- **`FT.ALIASLIST`** ([#3925](https://github.com/redis/go-redis/pull/3925)), **`COLLECT` reducer for `FT.AGGREGATE`** ([#3886](https://github.com/redis/go-redis/pull/3886)), **`RERANK` on HNSW vector fields in `FT.CREATE`** ([#3927](https://github.com/redis/go-redis/pull/3927)), and **`FT.HYBRID` timeout warnings** ([#3911](https://github.com/redis/go-redis/pull/3911)) — search coverage.
|
||||
|
||||
### Cross-SDK Aligned Defaults
|
||||
|
||||
Default configuration values now follow the cross-SDK configuration proposal shared by all Redis client libraries ([#3918](https://github.com/redis/go-redis/pull/3918)):
|
||||
|
||||
| Setting | Old default | New default |
|
||||
|---|---|---|
|
||||
| `ReadTimeout` / `WriteTimeout` | 3s | 5s |
|
||||
| Retry backoff (min/max) | 8ms / 512ms | 10ms / 1s |
|
||||
| Cluster state reload interval | 10s | 60s |
|
||||
| TCP keep-alive | 5min period | 30s idle / 5s interval / 3 probes (`net.KeepAliveConfig`) |
|
||||
|
||||
Applications that set these values explicitly are unaffected; applications relying on the old defaults inherit the new ones.
|
||||
|
||||
### Data-Race and Parser Hardening Sweep
|
||||
|
||||
A systematic audit fixed data races across the client — hooks (`AddHook`, [#3868](https://github.com/redis/go-redis/pull/3868)), `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), cluster node slices ([#3861](https://github.com/redis/go-redis/pull/3861)), pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), maintenance notifications ([#3894](https://github.com/redis/go-redis/pull/3894), [#3872](https://github.com/redis/go-redis/pull/3872)), pool handoff ([#3876](https://github.com/redis/go-redis/pull/3876)), and `redisotel` ([#3881](https://github.com/redis/go-redis/pull/3881)) — and hardened the RESP parsers against malformed or unexpected replies: over-reads on nil replies ([#3874](https://github.com/redis/go-redis/pull/3874)), integer overflow when skipping map/attribute bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), unhashable RESP3 map keys ([#3873](https://github.com/redis/go-redis/pull/3873)), odd-length flat replies ([#3900](https://github.com/redis/go-redis/pull/3900)), mismatched declared array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), unexpected extra reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), and nil elements in numeric/bool slice replies ([#3922](https://github.com/redis/go-redis/pull/3922)).
|
||||
|
||||
### PubSub `Receive` Hang Fix
|
||||
|
||||
`PeekPushNotificationName` blocked until 36 bytes were buffered, so a short subscribe confirmation (channel name of six or fewer characters) on an otherwise idle connection hung `PubSub.Receive` forever — a regression introduced in 9.20.1 by [#3842](https://github.com/redis/go-redis/pull/3842). The peek now parses whatever is already buffered and only waits for one more byte when the frame prefix is valid but incomplete. Fixes [#3935](https://github.com/redis/go-redis/issues/3935).
|
||||
|
||||
([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov)
|
||||
|
||||
### Correct Cluster Transaction Retries
|
||||
|
||||
The cluster transaction pipeline treated a `MULTI`...`EXEC` block as independently retryable commands, which could scatter a transaction across nodes or send malformed transactions on retry. Redirects (`MOVED`/`ASK`/`TRYAGAIN`) and aborts are now handled at the whole-transaction level, matching Redis transaction semantics: the transaction is re-routed and retried as a unit, never partially ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs).
|
||||
|
||||
### Credential Redaction in Command Tracing
|
||||
|
||||
`rediscmd.AppendCmd` — used by `redisotel` and `rediscensus` to render commands into span attributes — now redacts credential arguments as `<redacted>`: `AUTH`, `HELLO ... AUTH`, `CONFIG SET` of `requirepass` / `masterauth` / TLS key passphrases, `ACL SETUSER` password rules, and `MIGRATE ... AUTH`/`AUTH2`. The client sends `HELLO ... AUTH` on every handshake and `AUTH` on every streaming-credentials rotation through the regular hook chain, so tracing hooks previously captured credentials even when the application never issued an auth command itself ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e).
|
||||
|
||||
## ✨ New Features
|
||||
|
||||
- **`HIMPORT` command family**: `HImportPrepare` / `HImportSet` / `HImportDiscard` / `HImportDiscardAll` with lazy per-connection fieldset prepare replay ([#3919](https://github.com/redis/go-redis/pull/3919)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`LMOVEM` / `BLMOVEM`**: move multiple list elements in one call, with `COUNT` (up to N) or `EXACTLY` (all-or-nothing) semantics via `LMoveMArgs` ([#3913](https://github.com/redis/go-redis/pull/3913)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`SUnionCard` / `SDiffCard`**: cardinality of set union/difference ([#3897](https://github.com/redis/go-redis/pull/3897)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`XRead` / `XReadGroup` `MAXCOUNT` / `MAXSIZE`**: bound stream read responses by entry count or payload size ([#3898](https://github.com/redis/go-redis/pull/3898)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`TS.READ`**: read samples from a series starting at a given timestamp, with `TSReadEarliest` (`-`), `TSReadLatest` (`+`), and `TSReadNew` (`$`) sentinels ([#3896](https://github.com/redis/go-redis/pull/3896)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`TS.QUERYLABELS`**: query label names/values across time series ([#3926](https://github.com/redis/go-redis/pull/3926)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`TS.NRANGE` / `TS.NREVRANGE`**: range queries across multiple series ([#3870](https://github.com/redis/go-redis/pull/3870)) by [@ofekshenawa](https://github.com/ofekshenawa), with multiple aggregators per key ([#3937](https://github.com/redis/go-redis/pull/3937)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`TS.MRANGE` / `TS.MREVRANGE` `EXCLUDEEMPTY`**: skip series with no samples in the result ([#3912](https://github.com/redis/go-redis/pull/3912)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`FT.ALIASLIST`**: list all index aliases ([#3925](https://github.com/redis/go-redis/pull/3925)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`FT.AGGREGATE` `COLLECT` reducer**: collect grouped values into an array ([#3886](https://github.com/redis/go-redis/pull/3886)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`FT.CREATE` `RERANK`**: `RERANK` parameter on HNSW vector field definitions ([#3927](https://github.com/redis/go-redis/pull/3927)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`FT.HYBRID` timeout warnings**: timeout warnings are now populated in hybrid search results ([#3911](https://github.com/redis/go-redis/pull/3911)) by [@ofekshenawa](https://github.com/ofekshenawa)
|
||||
- **`FT.HYBRID` KNN `SHARD_K_RATIO`** (Redis 8.8+): per-shard K ratio for KNN clauses ([#3841](https://github.com/redis/go-redis/pull/3841)) by [@ndyakov](https://github.com/ndyakov)
|
||||
|
||||
## 🐛 Bug Fixes
|
||||
|
||||
- **PubSub `Receive` hang**: peek push-notification names without demanding 36 buffered bytes, fixing a hang on short subscribe confirmations (fixes [#3935](https://github.com/redis/go-redis/issues/3935), regression from 9.20.1) ([#3936](https://github.com/redis/go-redis/pull/3936)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **Cluster transactions**: re-route the whole tx pipeline on redirect/abort instead of per-command ([#3909](https://github.com/redis/go-redis/pull/3909)) by [@cxljs](https://github.com/cxljs)
|
||||
- **Credential leak in traces**: `rediscmd.AppendCmd` redacts credential arguments (`AUTH`, `HELLO ... AUTH`, `CONFIG SET` secret params, `ACL SETUSER` password rules, `MIGRATE AUTH`/`AUTH2`), so `redisotel` / `rediscensus` span attributes no longer contain passwords ([#3939](https://github.com/redis/go-redis/pull/3939)) by [@saddamr3e](https://github.com/saddamr3e)
|
||||
- **`WaitAOF` return type**: returns `*IntSliceCmd` matching the two-integer `WAITAOF` reply ([#3888](https://github.com/redis/go-redis/pull/3888)) by [@CipherN9](https://github.com/CipherN9)
|
||||
- **`Ring.Publish` routing**: publish to the shard that owns the topic instead of a round-robined one ([#3893](https://github.com/redis/go-redis/pull/3893)) by [@dkindel](https://github.com/dkindel)
|
||||
- **Pool `OnRemove` hooks**: fire `OnRemove` on `putConn` eviction paths so removal hooks see every evicted connection ([#3932](https://github.com/redis/go-redis/pull/3932)) by [@cxljs](https://github.com/cxljs)
|
||||
- **`UniversalClient` `InfoMap`**: added `InfoMap` to the `Cmdable` interface ([#3904](https://github.com/redis/go-redis/pull/3904)) by [@nazarli-shabnam](https://github.com/nazarli-shabnam)
|
||||
- **`SlowLogGet` context**: pass the caller's context instead of a background one ([#3915](https://github.com/redis/go-redis/pull/3915)) by [@sonnemusk](https://github.com/sonnemusk)
|
||||
- **`ModuleLoadex` nil config**: return an error instead of panicking on nil config ([#3916](https://github.com/redis/go-redis/pull/3916)) by [@sonnemusk](https://github.com/sonnemusk)
|
||||
- **`ParseURL` IPv6 hosts**: keep single brackets for IPv6 hosts without a port ([#3882](https://github.com/redis/go-redis/pull/3882)) by [@sueun-dev](https://github.com/sueun-dev)
|
||||
- **`ParseURL` durations**: treat unit durations `<= 0` as disabled ([#3866](https://github.com/redis/go-redis/pull/3866)) by [@sueun-dev](https://github.com/sueun-dev)
|
||||
- **Nil `*uint8` encoding**: encode nil `*uint8` as `"0"` like other numeric pointers ([#3869](https://github.com/redis/go-redis/pull/3869)) by [@sueun-dev](https://github.com/sueun-dev)
|
||||
- **`JSONSliceCmd` read errors**: return the read error from `readReply` instead of swallowing it ([#3903](https://github.com/redis/go-redis/pull/3903)) by [@saddamr3e](https://github.com/saddamr3e)
|
||||
- **RESP parser hardening**: reconcile declared entry-array lengths ([#3907](https://github.com/redis/go-redis/pull/3907)), handle nil elements in int/uint/bool slice parsers ([#3922](https://github.com/redis/go-redis/pull/3922)), drain unexpected reply frames ([#3884](https://github.com/redis/go-redis/pull/3884)), reject odd-length flat replies in Z/KeyValue parsers ([#3900](https://github.com/redis/go-redis/pull/3900)), avoid int overflow when skipping map/attr bodies ([#3877](https://github.com/redis/go-redis/pull/3877)), don't over-read nil replies in `Reader.Discard` ([#3874](https://github.com/redis/go-redis/pull/3874)) by [@saddamr3e](https://github.com/saddamr3e); reject unhashable keys in RESP3 map parsing ([#3873](https://github.com/redis/go-redis/pull/3873)) by [@iabdullah215](https://github.com/iabdullah215)
|
||||
- **Data races**: hook state during `AddHook` ([#3868](https://github.com/redis/go-redis/pull/3868)), `onNewNode` during `Ring.SetAddrs` ([#3862](https://github.com/redis/go-redis/pull/3862)), shared masters/slaves slices in cluster ([#3861](https://github.com/redis/go-redis/pull/3861)), shared `opt.Addr` during pub/sub reconnect ([#3906](https://github.com/redis/go-redis/pull/3906)), `clusterStateReloadCallback` in maintnotifications ([#3894](https://github.com/redis/go-redis/pull/3894)), conn reader in `isHealthyConn` during handoff ([#3876](https://github.com/redis/go-redis/pull/3876)) by [@saddamr3e](https://github.com/saddamr3e); handoff race window in maintnotifications ([#3872](https://github.com/redis/go-redis/pull/3872)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **`redisotel`**: use `ObservableCounter` for cumulative pool stats ([#3914](https://github.com/redis/go-redis/pull/3914)) by [@Solaris-star](https://github.com/Solaris-star); avoid a data race on shared attributes during `MinIdleConns` warmup ([#3881](https://github.com/redis/go-redis/pull/3881)) by [@ndyakov](https://github.com/ndyakov)
|
||||
|
||||
## 🧰 Maintenance
|
||||
|
||||
- **Cross-SDK default alignment**: new defaults for timeouts, retry backoff, cluster state reload, and TCP keep-alive ([#3918](https://github.com/redis/go-redis/pull/3918)) by [@ndyakov](https://github.com/ndyakov)
|
||||
- **CI on Redis 8.10**: 8.10 made the default test version ([#3920](https://github.com/redis/go-redis/pull/3920)) with version gating by major.minor ([#3908](https://github.com/redis/go-redis/pull/3908)) by [@ofekshenawa](https://github.com/ofekshenawa); the test stack now runs the GA `redislabs/client-libs-test:8.10.0` image and 8.8 was dropped from the CI matrix ([#3940](https://github.com/redis/go-redis/pull/3940))
|
||||
- **Type-safe atomics**: use typed `sync/atomic` value types ([#3860](https://github.com/redis/go-redis/pull/3860)) and remove the dead `assertUnstableCommand` RESP3 path ([#3928](https://github.com/redis/go-redis/pull/3928)) by [@cxljs](https://github.com/cxljs)
|
||||
- **Docs**: clarify that `ExpireTime` / `PExpireTime` return Unix timestamps ([#3917](https://github.com/redis/go-redis/pull/3917)) by [@sonnemusk](https://github.com/sonnemusk); remove a duplicate example step ([#3875](https://github.com/redis/go-redis/pull/3875)) by [@andy-stark-redis](https://github.com/andy-stark-redis)
|
||||
|
||||
## 👥 Contributors
|
||||
|
||||
We'd like to thank all the contributors who worked on this release!
|
||||
|
||||
[@andy-stark-redis](https://github.com/andy-stark-redis), [@CipherN9](https://github.com/CipherN9), [@cxljs](https://github.com/cxljs), [@dkindel](https://github.com/dkindel), [@iabdullah215](https://github.com/iabdullah215), [@nazarli-shabnam](https://github.com/nazarli-shabnam), [@ndyakov](https://github.com/ndyakov), [@ofekshenawa](https://github.com/ofekshenawa), [@saddamr3e](https://github.com/saddamr3e), [@Solaris-star](https://github.com/Solaris-star), [@sonnemusk](https://github.com/sonnemusk), [@sueun-dev](https://github.com/sueun-dev)
|
||||
|
||||
---
|
||||
|
||||
**Full Changelog**: https://github.com/redis/go-redis/compare/v9.21.0...v9.22.0-beta.1
|
||||
|
||||
# 9.21.0 (2026-06-18)
|
||||
|
||||
This is a minor release adding new features and bug fixes. There are no breaking changes; upgrading from 9.20.x is a drop-in replacement.
|
||||
|
||||
+2609
File diff suppressed because it is too large
Load Diff
+128
@@ -0,0 +1,128 @@
|
||||
# Autopipelining benchmarks
|
||||
|
||||
These benchmarks validate the goal of autopipelining: batching concurrent
|
||||
commands into pipelines cuts network round-trips and raises throughput, without
|
||||
callers writing pipeline code.
|
||||
|
||||
Two measurement rules keep every number honest:
|
||||
|
||||
- **Throughput is measured on executed commands** — a command is counted only
|
||||
after its result has been read (`.Result()` / `.Err()`), never when it is
|
||||
merely queued. On the deferred face `ap.Set` returns immediately, so counting
|
||||
calls would measure enqueue speed, not throughput.
|
||||
- **Rates divide by the timed region, not the nominal window.** The
|
||||
fixed-duration drivers keep draining whatever was in flight when the deadline
|
||||
hit; that drain is part of the timed region, so `ops/sec` cannot be inflated
|
||||
by work that finished after the window closed.
|
||||
|
||||
## Running
|
||||
|
||||
The benchmarks talk to a real Redis on `:6379` (they skip when none answers).
|
||||
Start one first:
|
||||
|
||||
```sh
|
||||
make docker.start # or: docker run --rm -p 6379:6379 redis
|
||||
|
||||
# the headline three-way throughput comparison:
|
||||
go test -run '^$' -bench BenchmarkAutoPipelineThroughput -benchtime=1x .
|
||||
|
||||
# everything:
|
||||
go test -run '^$' -bench Benchmark -benchmem -benchtime=1x .
|
||||
```
|
||||
|
||||
The throughput benchmarks run for a fixed wall-clock duration (~3s each) and
|
||||
report `ops/sec`, so `-benchtime=1x` (one iteration) is correct for them.
|
||||
**Do not pass a time-based `-benchtime`** (e.g. `-benchtime=5s`): the
|
||||
fixed-duration drivers ignore `b.N`, so Go's framework would re-run the full
|
||||
window geometrically trying to fill the time budget.
|
||||
|
||||
The per-operation benchmarks (`BenchmarkDispatchPath`,
|
||||
`BenchmarkAutoPipelineZeroCopy`) are the opposite: their ns/op and allocs/op
|
||||
are only meaningful at the **default** `-benchtime` — at `-benchtime=1x` the
|
||||
workers still issue a minimum window each, all billed to a single iteration.
|
||||
(The "everything" command above also sweeps the repo's other root-package
|
||||
benchmarks; that is harmless, just broader than this file.)
|
||||
|
||||
## The headline benchmark: BenchmarkAutoPipelineThroughput
|
||||
|
||||
Three ways to issue the same workload (2000 goroutines), each counting only
|
||||
executed commands:
|
||||
|
||||
1. **Normal** — a plain client; each `Set` is a blocking round-trip. Bounded by
|
||||
Redis's non-pipelined ceiling (like `redis-benchmark` without `-P`).
|
||||
2. **AutoPipelineBlocking** — the blocking face with a parallel-batch config:
|
||||
`ap.Set(...)` blocks until executed, the same call shape as a normal client.
|
||||
Only one command per caller is in flight, but the flusher batches across the
|
||||
2000 callers into deep pipelines.
|
||||
3. **AutoPipelineWindowed** — the deferred face: each caller submits a window of
|
||||
200 commands, then reads the results. Keeps pipelines deepest.
|
||||
|
||||
The `WindowedGET` variant repeats (3) with GET instead of SET: SET throughput is
|
||||
server-bound (Redis's write processing), GET is cheaper on the server, so the
|
||||
GET number shows the client machinery itself is not the limit.
|
||||
|
||||
**Absolute numbers are machine- and load-dependent and vary a lot** — CPU
|
||||
count, Redis's own ceiling, network path (loopback vs docker veth vs real
|
||||
network), and noisy neighbors all move them by integer factors. The signal is
|
||||
the WITHIN-RUN multiplier against the `Normal` baseline measured in the same
|
||||
environment, plus `allocs/op` (which is exact and stable everywhere):
|
||||
|
||||
| variant | vs Normal (same run) |
|
||||
| ----------------------- | ---------------------- |
|
||||
| Normal | 1x (the baseline) |
|
||||
| AutoPipelineBlocking | ~10x |
|
||||
| AutoPipelineWindowed | ~25-30x |
|
||||
| AutoPipelineWindowedGET | above Windowed (reads) |
|
||||
|
||||
As one concrete example: an Apple Silicon laptop with a loopback Redis puts
|
||||
`Normal` around 80k ops/sec (so ~800k blocking, ~2.5M windowed); a 4-vCPU CI
|
||||
runner with dockerized Redis lands near half that on the CPU-bound variants —
|
||||
different absolutes, same multipliers and ordering.
|
||||
|
||||
The autopipeline variants use an explicit parallel-batch config
|
||||
(`MaxBatchSize: 300, MaxConcurrentBatches: 80, Unordered: true`) — **not the
|
||||
ordered default**. The default (`MaxConcurrentBatches: 1`,
|
||||
`DefaultAutoPipelineOptions` / `DefaultBlockingAutoPipelineOptions`) serializes
|
||||
batch execution: blocking usage lands at roughly half the parallel-batch
|
||||
multiplier, while windowed submission stays well into the tens-of-x even
|
||||
ordered.
|
||||
|
||||
## The other benchmarks
|
||||
|
||||
- **BenchmarkIndividualCommands** — plain-client baseline: one blocking
|
||||
round-trip per command across GOMAXPROCS workers. Its ns/op is your
|
||||
environment's RTT floor; every other number is best read against it.
|
||||
- **BenchmarkManualPipeline** — hand-built 100-deep `Pipeline().Exec()`,
|
||||
sequential: the per-command cost of explicit pipelining (roughly a tenth
|
||||
of a round-trip per command). The ceiling autopipelining approaches
|
||||
without anyone writing pipeline code.
|
||||
- **BenchmarkDispatchPath** — the engine's per-command dispatch cost with
|
||||
honest `b.N` accounting: ns/op and allocs/op per executed command
|
||||
(4 allocs/cmd on the submit path; unordered dispatch roughly halves the
|
||||
ordered ns/op), plus the lone-command blocking fast path (~1 RTT).
|
||||
- **BenchmarkFutureFace** — the typed future face on the ordered default
|
||||
config: per-command reads (`InOrder`) vs windowed reads (`Window200`,
|
||||
roughly 2x InOrder).
|
||||
- **BenchmarkAutoPipelineSubmit** — the non-blocking `Submit` entry point,
|
||||
windowed, on the ordered default; lands in the same band as
|
||||
`FutureFace/Window200`.
|
||||
- **BenchmarkAutoPipelineZeroCopy** — `GetToBuffer`/`SetFromBuffer` vs regular
|
||||
`Get`/`Set` (Set+Get pairs): B/op drops ~10x at 4KiB and ~90x at 64KiB
|
||||
(payloads decode into the caller's buffer instead of fresh strings), with
|
||||
throughput at parity or better; allocs/op 10 vs 11. The B/op and allocs/op
|
||||
ratios are environment-independent.
|
||||
- **BenchmarkClusterAutoPipelineThroughput** — the same blocking/windowed
|
||||
drivers against a local 3-master cluster (slot-routed shard batches keep
|
||||
per-node pipelines deep; scales past the standalone numbers in the same
|
||||
environment, with a wide run-order-dependent spread). Skips when no
|
||||
cluster answers on `:16600-16602`.
|
||||
|
||||
## What was deliberately removed
|
||||
|
||||
Earlier revisions carried "tuning sweep" benchmarks (batch sizes, flush
|
||||
delays, buffer sizes) whose numbers were dominated by the configured
|
||||
`MaxFlushDelay` timer at low parallelism — every swept value reported the same
|
||||
timer readout, which could only mislead someone tuning from them. They were
|
||||
removed rather than fixed: `BenchmarkDispatchPath` and the throughput drivers
|
||||
cover the engine's real knobs. Tune with your own workload shape; the engine's
|
||||
defaults need no tuning to hit the numbers above.
|
||||
+512
-23
File diff suppressed because it is too large
Load Diff
+27
@@ -109,6 +109,13 @@ var defaultPolicies = map[module]map[commandName]*routing.CommandPolicy{
|
||||
Request: routing.ReqDefault,
|
||||
Response: routing.RespDefaultKeyless,
|
||||
},
|
||||
"aliaslist": {
|
||||
Request: routing.ReqDefault,
|
||||
Response: routing.RespDefaultKeyless,
|
||||
Tips: map[string]string{
|
||||
routing.ReadOnlyCMD: "",
|
||||
},
|
||||
},
|
||||
"info": {
|
||||
Request: routing.ReqDefault,
|
||||
Response: routing.RespDefaultKeyless,
|
||||
@@ -156,6 +163,26 @@ var defaultPolicies = map[module]map[commandName]*routing.CommandPolicy{
|
||||
},
|
||||
}
|
||||
|
||||
// defaultPolicyKeyless reports whether name (e.g. "ft.aliaslist") is registered
|
||||
// in the static policy table as a plain keyless command: default request
|
||||
// routing with a keyless response policy. Commands whose slot comes from a key
|
||||
// (RespDefaultHashSlot, e.g. ft.suglen) or with special request routing
|
||||
// (ReqSpecial, e.g. ft.cursor) are excluded — their key position must still be
|
||||
// resolved. cmdFirstKeyPosWithInfo consults this so the initial slot
|
||||
// computation on a cold command-info cache matches the policy the router
|
||||
// applies once the command reaches routeAndRun.
|
||||
func defaultPolicyKeyless(name string) bool {
|
||||
i := strings.IndexByte(name, '.')
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
policy, ok := defaultPolicies[name[:i]][name[i+1:]]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return policy.Request == routing.ReqDefault && policy.Response == routing.RespDefaultKeyless
|
||||
}
|
||||
|
||||
type CommandInfoResolveFunc func(ctx context.Context, cmd Cmder) *routing.CommandPolicy
|
||||
|
||||
type commandInfoResolver struct {
|
||||
|
||||
+101
-3
@@ -199,6 +199,9 @@ type Cmdable interface {
|
||||
ClientUnblock(ctx context.Context, id int64) *IntCmd
|
||||
ClientUnblockWithError(ctx context.Context, id int64) *IntCmd
|
||||
ClientMaintNotifications(ctx context.Context, enabled bool, endpointType string) *StatusCmd
|
||||
ClientTracking(ctx context.Context, on bool, opt *ClientTrackingOptions) *StatusCmd
|
||||
ClientTrackingOn(ctx context.Context, opt *ClientTrackingOptions) *StatusCmd
|
||||
ClientTrackingOff(ctx context.Context) *StatusCmd
|
||||
ConfigGet(ctx context.Context, parameter string) *MapStringStringCmd
|
||||
ConfigResetStat(ctx context.Context) *StatusCmd
|
||||
ConfigSet(ctx context.Context, parameter, value string) *StatusCmd
|
||||
@@ -209,6 +212,7 @@ type Cmdable interface {
|
||||
FlushDB(ctx context.Context) *StatusCmd
|
||||
FlushDBAsync(ctx context.Context) *StatusCmd
|
||||
Info(ctx context.Context, section ...string) *StringCmd
|
||||
InfoMap(ctx context.Context, section ...string) *InfoCmd
|
||||
LastSave(ctx context.Context) *IntCmd
|
||||
Save(ctx context.Context) *StatusCmd
|
||||
Shutdown(ctx context.Context) *StatusCmd
|
||||
@@ -296,8 +300,8 @@ func (c cmdable) Wait(ctx context.Context, numSlaves int, timeout time.Duration)
|
||||
return cmd
|
||||
}
|
||||
|
||||
func (c cmdable) WaitAOF(ctx context.Context, numLocal, numSlaves int, timeout time.Duration) *IntCmd {
|
||||
cmd := NewIntCmd(ctx, "waitAOF", numLocal, numSlaves, int(timeout/time.Millisecond))
|
||||
func (c cmdable) WaitAOF(ctx context.Context, numLocal, numSlaves int, timeout time.Duration) *IntSliceCmd {
|
||||
cmd := NewIntSliceCmd(ctx, "waitAOF", numLocal, numSlaves, int(timeout/time.Millisecond))
|
||||
cmd.setReadTimeout(timeout)
|
||||
_ = c(ctx, cmd)
|
||||
return cmd
|
||||
@@ -565,6 +569,95 @@ func (c cmdable) ClientMaintNotifications(ctx context.Context, enabled bool, end
|
||||
return cmd
|
||||
}
|
||||
|
||||
// ClientTrackingOptions configures CLIENT TRACKING ON. See
|
||||
// https://redis.io/commands/client-tracking/ for semantics.
|
||||
type ClientTrackingOptions struct {
|
||||
Redirect int64
|
||||
Bcast bool
|
||||
Prefixes []string
|
||||
OptIn bool
|
||||
OptOut bool
|
||||
NoLoop bool
|
||||
}
|
||||
|
||||
// ClientTracking enables or disables server-assisted client-side caching for
|
||||
// the ONE connection that happens to serve this command. On a pooled client
|
||||
// that connection is arbitrary, so this is only meaningful on a dedicated
|
||||
// connection (see Client.Conn). When on is false, opt is ignored. Invalid
|
||||
// option combinations are reported via the returned command's Err and nothing
|
||||
// is sent to the server.
|
||||
//
|
||||
// Must not be combined with the built-in client-side cache: on a client
|
||||
// configured with Options.ClientSideCache or ClientSideCacheConfig this
|
||||
// command is rejected, because changing a pool connection's tracking state
|
||||
// would silently break the cache's invalidation.
|
||||
func (c cmdable) ClientTracking(ctx context.Context, on bool, opt *ClientTrackingOptions) *StatusCmd {
|
||||
if !on {
|
||||
return c.ClientTrackingOff(ctx)
|
||||
}
|
||||
return c.ClientTrackingOn(ctx, opt)
|
||||
}
|
||||
|
||||
// ClientTrackingOn enables tracking on the serving connection. See
|
||||
// ClientTracking for the pooled-client and built-in-CSC caveats.
|
||||
func (c cmdable) ClientTrackingOn(ctx context.Context, opt *ClientTrackingOptions) *StatusCmd {
|
||||
args := []interface{}{"client", "tracking", "on"}
|
||||
if opt != nil {
|
||||
if err := validateClientTrackingOptions(opt); err != nil {
|
||||
cmd := NewStatusCmd(ctx, args...)
|
||||
cmd.SetErr(err)
|
||||
return cmd
|
||||
}
|
||||
args = appendClientTrackingOptions(args, opt)
|
||||
}
|
||||
cmd := NewStatusCmd(ctx, args...)
|
||||
_ = c(ctx, cmd)
|
||||
return cmd
|
||||
}
|
||||
|
||||
// ClientTrackingOff disables tracking on the serving connection. See
|
||||
// ClientTracking for the pooled-client and built-in-CSC caveats.
|
||||
func (c cmdable) ClientTrackingOff(ctx context.Context) *StatusCmd {
|
||||
cmd := NewStatusCmd(ctx, "client", "tracking", "off")
|
||||
_ = c(ctx, cmd)
|
||||
return cmd
|
||||
}
|
||||
|
||||
func validateClientTrackingOptions(opt *ClientTrackingOptions) error {
|
||||
if opt.OptIn && opt.OptOut {
|
||||
return errors.New("redis: CLIENT TRACKING OPTIN and OPTOUT are mutually exclusive")
|
||||
}
|
||||
if opt.Bcast && (opt.OptIn || opt.OptOut) {
|
||||
return errors.New("redis: CLIENT TRACKING BCAST cannot be combined with OPTIN or OPTOUT")
|
||||
}
|
||||
if len(opt.Prefixes) > 0 && !opt.Bcast {
|
||||
return errors.New("redis: CLIENT TRACKING PREFIX requires BCAST")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendClientTrackingOptions(args []interface{}, opt *ClientTrackingOptions) []interface{} {
|
||||
if opt.Redirect != 0 {
|
||||
args = append(args, "redirect", opt.Redirect)
|
||||
}
|
||||
if opt.Bcast {
|
||||
args = append(args, "bcast")
|
||||
}
|
||||
for _, p := range opt.Prefixes {
|
||||
args = append(args, "prefix", p)
|
||||
}
|
||||
if opt.OptIn {
|
||||
args = append(args, "optin")
|
||||
}
|
||||
if opt.OptOut {
|
||||
args = append(args, "optout")
|
||||
}
|
||||
if opt.NoLoop {
|
||||
args = append(args, "noloop")
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
func (c cmdable) ConfigGet(ctx context.Context, parameter string) *MapStringStringCmd {
|
||||
@@ -706,7 +799,7 @@ func (c cmdable) ReplicaOf(ctx context.Context, host, port string) *StatusCmd {
|
||||
}
|
||||
|
||||
func (c cmdable) SlowLogGet(ctx context.Context, num int64) *SlowLogCmd {
|
||||
cmd := NewSlowLogCmd(context.Background(), "slowlog", "get", num)
|
||||
cmd := NewSlowLogCmd(ctx, "slowlog", "get", num)
|
||||
_ = c(ctx, cmd)
|
||||
return cmd
|
||||
}
|
||||
@@ -797,6 +890,11 @@ func (c *ModuleLoadexConfig) toArgs() []interface{} {
|
||||
|
||||
// ModuleLoadex Redis `MODULE LOADEX path [CONFIG name value [CONFIG name value ...]] [ARGS args [args ...]]` command.
|
||||
func (c cmdable) ModuleLoadex(ctx context.Context, conf *ModuleLoadexConfig) *StringCmd {
|
||||
if conf == nil {
|
||||
cmd := NewStringCmd(ctx)
|
||||
cmd.SetErr(errors.New("redis: ModuleLoadex nil config"))
|
||||
return cmd
|
||||
}
|
||||
cmd := NewStringCmd(ctx, conf.toArgs()...)
|
||||
_ = c(ctx, cmd)
|
||||
return cmd
|
||||
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
package redis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/redis/go-redis/v9/internal/proto"
|
||||
)
|
||||
|
||||
// defaultCacheableCommands is the allow-list of read-only, deterministic
|
||||
// commands whose responses may be stored in the client-side cache. Keys are
|
||||
// lowercase to match baseCmd.Name() on the hot path.
|
||||
var defaultCacheableCommands = map[string]struct{}{
|
||||
// String commands
|
||||
"get": {}, "mget": {}, "getbit": {}, "getrange": {},
|
||||
"strlen": {}, "substr": {},
|
||||
// Hash commands
|
||||
"hget": {}, "hgetall": {}, "hmget": {},
|
||||
"hkeys": {}, "hvals": {}, "hlen": {},
|
||||
"hexists": {}, "hstrlen": {},
|
||||
// List commands
|
||||
"lindex": {}, "llen": {}, "lpos": {}, "lrange": {},
|
||||
// Set commands
|
||||
"scard": {}, "sismember": {}, "smembers": {}, "smismember": {},
|
||||
"sdiff": {}, "sinter": {}, "sintercard": {}, "sunion": {},
|
||||
// Sorted-set commands
|
||||
"zcard": {}, "zcount": {}, "zlexcount": {}, "zmscore": {},
|
||||
"zrange": {}, "zrangebylex": {}, "zrangebyscore": {},
|
||||
"zrank": {}, "zrevrange": {}, "zrevrangebylex": {},
|
||||
"zrevrangebyscore": {}, "zrevrank": {}, "zscore": {},
|
||||
"zdiff": {}, "zinter": {}, "zunion": {},
|
||||
// Bit commands
|
||||
"bitcount": {}, "bitfield_ro": {}, "bitpos": {},
|
||||
// Key/generic commands
|
||||
"exists": {}, "type": {}, "sort_ro": {}, "lcs": {},
|
||||
// Geo commands
|
||||
"geodist": {}, "geohash": {}, "geopos": {}, "geosearch": {},
|
||||
"georadiusbymember_ro": {}, "georadius_ro": {},
|
||||
// Stream commands. XREAD is deliberately excluded: it supports BLOCK, and
|
||||
// its $/+ IDs are state-relative, so identical args are not deterministic.
|
||||
// XPENDING is excluded for the same class of reason: its extended form
|
||||
// returns wall-clock-relative idle times and its IDLE filter is
|
||||
// time-dependent, so identical args yield different correct results with
|
||||
// no key modification (and therefore no invalidation).
|
||||
"xlen": {}, "xrange": {}, "xrevrange": {},
|
||||
// JSON (RedisJSON) commands
|
||||
"json.get": {}, "json.mget": {}, "json.arrindex": {}, "json.arrlen": {},
|
||||
"json.objkeys": {}, "json.objlen": {}, "json.resp": {},
|
||||
"json.strlen": {}, "json.type": {},
|
||||
// TimeSeries commands
|
||||
"ts.get": {}, "ts.info": {}, "ts.range": {}, "ts.revrange": {},
|
||||
}
|
||||
|
||||
// isCacheable reports whether cmd is eligible for client-side caching: its
|
||||
// name is on the allow-list and it operates on at least one key.
|
||||
func isCacheable(cmd Cmder) bool {
|
||||
// Commands such as RawWriteToCmd stream replies directly to an io.Writer.
|
||||
// Capturing their replies for CSC would buffer the entire response first,
|
||||
// defeating their streaming and allocation guarantees.
|
||||
if cmd.NoRetry() {
|
||||
return false
|
||||
}
|
||||
if _, ok := defaultCacheableCommands[cmd.Name()]; !ok {
|
||||
return false
|
||||
}
|
||||
// SORT_RO ... BY/GET reads pattern keys that extractRedisKeys can't
|
||||
// enumerate, so its invalidations would be dropped and the result go stale.
|
||||
// Plain SORT_RO is fine.
|
||||
if cmd.Name() == "sort_ro" && sortROHasByGet(cmd) {
|
||||
return false
|
||||
}
|
||||
return cmdFirstKeyPosWithInfo(cmd, nil) != 0
|
||||
}
|
||||
|
||||
// sortROHasByGet reports whether a SORT_RO invocation uses BY or GET
|
||||
// (case-insensitive), scanning past the command name and key. stringArg
|
||||
// normalizes string, *string, and []byte tokens.
|
||||
func sortROHasByGet(cmd Cmder) bool {
|
||||
for i := 2; i < len(cmd.Args()); i++ {
|
||||
if s := cmd.stringArg(i); strings.EqualFold(s, "by") || strings.EqualFold(s, "get") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isClientTrackingCmd reports whether cmd is a CLIENT TRACKING subcommand (any
|
||||
// mode: ON, OFF, or with options). Name and stringArg normalize string,
|
||||
// *string, and []byte arguments.
|
||||
func isClientTrackingCmd(cmd Cmder) bool {
|
||||
return cmd.Name() == "client" && strings.EqualFold(cmd.stringArg(1), "tracking")
|
||||
}
|
||||
|
||||
// isSelectCmd reports whether cmd changes the selected database on its
|
||||
// connection. CSC keys are namespaced with Options.DB, so a runtime SELECT
|
||||
// would make the connection's actual database diverge from the cache namespace.
|
||||
func isSelectCmd(cmd Cmder) bool {
|
||||
return cmd.Name() == "select"
|
||||
}
|
||||
|
||||
// isAuthCmd reports whether cmd changes the authenticated user on its
|
||||
// connection. The cache namespace is fixed from Options.Username, so runtime
|
||||
// authentication would make the connection identity diverge from it.
|
||||
func isAuthCmd(cmd Cmder) bool {
|
||||
return cmd.Name() == "auth"
|
||||
}
|
||||
|
||||
// isProtocolChangingHelloCmd reports whether HELLO includes a protocol version
|
||||
// (and can therefore switch a tracked RESP3 connection to RESP2). A bare HELLO
|
||||
// only reports connection properties and is safe.
|
||||
func isProtocolChangingHelloCmd(cmd Cmder) bool {
|
||||
return cmd.Name() == "hello" && len(cmd.Args()) > 1
|
||||
}
|
||||
|
||||
// isResetCmd reports whether cmd resets all server-side connection state.
|
||||
// RESET disables tracking, switches to RESP2, deauthenticates, and changes
|
||||
// other state that a pooled CSC connection relies on.
|
||||
func isResetCmd(cmd Cmder) bool {
|
||||
return cmd.Name() == "reset"
|
||||
}
|
||||
|
||||
// isSubscribeCmd reports whether a raw command would turn an ordinary pooled
|
||||
// connection into a Pub/Sub connection. Pub/Sub pushes are deliberately left
|
||||
// for the dedicated PubSub reader, so the CSC drainer cannot safely own such a
|
||||
// connection.
|
||||
func isSubscribeCmd(cmd Cmder) bool {
|
||||
switch cmd.Name() {
|
||||
case "subscribe", "psubscribe", "ssubscribe":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// buildCacheKey returns the RESP-encoded form of the command's argument list,
|
||||
// used as a collision-free canonical cache key. ok is false when the writer
|
||||
// cannot marshal the arguments, in which case the caller must skip caching
|
||||
// rather than bucket the command under an empty key.
|
||||
func buildCacheKey(cmd Cmder) (string, bool) {
|
||||
args := cmd.Args()
|
||||
if len(args) == 0 {
|
||||
return "", false
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := proto.NewWriter(&buf).WriteArgs(args); err != nil {
|
||||
return "", false
|
||||
}
|
||||
return buf.String(), true
|
||||
}
|
||||
|
||||
// keyArg renders the key argument at pos exactly as proto.Writer sends it to
|
||||
// the server, so invalidation lookups match the key names in the server's
|
||||
// "invalidate" pushes. Only types whose stringArg rendering is byte-identical
|
||||
// to the wire encoding are accepted (fmt.Sprint of any integer matches the
|
||||
// writer's base-10 strconv output); for anything else — pointers, bools,
|
||||
// times, durations, floats, BinaryMarshaler values — the rendering can
|
||||
// diverge, the invalidation would never match, and the entry would be served
|
||||
// stale forever, so ok=false and the caller skips caching (see processCached).
|
||||
func keyArg(cmd Cmder, pos int) (string, bool) {
|
||||
args := cmd.Args()
|
||||
if pos < 0 || pos >= len(args) {
|
||||
return "", false
|
||||
}
|
||||
switch args[pos].(type) {
|
||||
case string, []byte,
|
||||
int, int8, int16, int32, int64,
|
||||
uint, uint8, uint16, uint32, uint64:
|
||||
return cmd.stringArg(pos), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// extractRedisKeys returns the Redis key arguments from cmd. The result lets
|
||||
// the cache map incoming invalidations back to affected entries. Returns nil
|
||||
// (caller skips caching) when any key
|
||||
// argument cannot be rendered in its wire form (see keyArg).
|
||||
func extractRedisKeys(cmd Cmder) []string {
|
||||
firstKey := cmdFirstKeyPosWithInfo(cmd, nil)
|
||||
if firstKey == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
argsLen := len(cmd.Args())
|
||||
if firstKey >= argsLen {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch cmd.Name() {
|
||||
// All remaining args from firstKeyPos are keys.
|
||||
case "mget", "exists", "sdiff", "sinter", "sunion":
|
||||
keys := make([]string, 0, argsLen-firstKey)
|
||||
for i := firstKey; i < argsLen; i++ {
|
||||
k, ok := keyArg(cmd, i)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
|
||||
// Numkeys pattern: numkeys at args[1], keys from args[2].
|
||||
case "sintercard", "zdiff", "zinter", "zunion":
|
||||
if argsLen < 3 {
|
||||
return nil
|
||||
}
|
||||
numKeys, err := strconv.Atoi(cmd.stringArg(1))
|
||||
if err != nil || numKeys <= 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, numKeys)
|
||||
for i := 2; i < 2+numKeys && i < argsLen; i++ {
|
||||
k, ok := keyArg(cmd, i)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
|
||||
// LCS: exactly two consecutive keys starting at firstKeyPos.
|
||||
case "lcs":
|
||||
if firstKey+1 >= argsLen {
|
||||
return nil
|
||||
}
|
||||
k1, ok1 := keyArg(cmd, firstKey)
|
||||
k2, ok2 := keyArg(cmd, firstKey+1)
|
||||
if !ok1 || !ok2 {
|
||||
return nil
|
||||
}
|
||||
return []string{k1, k2}
|
||||
|
||||
// JSON.MGET: keys from firstKeyPos to second-to-last (last arg is the
|
||||
// JSON path, not a key).
|
||||
case "json.mget":
|
||||
lastKey := argsLen - 2
|
||||
if lastKey < firstKey {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, lastKey-firstKey+1)
|
||||
for i := firstKey; i <= lastKey; i++ {
|
||||
k, ok := keyArg(cmd, i)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Single key at firstKeyPos (GET, HGET, LRANGE, ...).
|
||||
k, ok := keyArg(cmd, firstKey)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return []string{k}
|
||||
}
|
||||
+922
@@ -0,0 +1,922 @@
|
||||
package redis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9/internal"
|
||||
"github.com/redis/go-redis/v9/internal/pool"
|
||||
"github.com/redis/go-redis/v9/internal/proto"
|
||||
"github.com/redis/go-redis/v9/push"
|
||||
)
|
||||
|
||||
// cscRegisterCleanups arranges for a client dropped without Close to stop its
|
||||
// background CSC drainer. The drainer's exit path revokes its pool's cache
|
||||
// coverage; the runtime cleanup itself stays non-blocking and never captures
|
||||
// *Client, so the wrapper remains collectible.
|
||||
func cscRegisterCleanups(c *Client) {
|
||||
h := c.baseClient.cscDrainHandle
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
// Capture cscActive (a standalone *atomic.Bool, not *Client) so the cleanup
|
||||
// also stops clones from serving once the drainer is gone.
|
||||
active := c.baseClient.cscActive
|
||||
runtime.AddCleanup(c, func(h *cscDrainHandle) {
|
||||
if active != nil {
|
||||
active.Store(false)
|
||||
}
|
||||
h.signalStop()
|
||||
}, h)
|
||||
}
|
||||
|
||||
// ClientSideCacheConfig configures the built-in client-side cache. Pass a
|
||||
// non-nil value to Options.ClientSideCacheConfig to enable caching on a RESP3
|
||||
// client.
|
||||
//
|
||||
// Experimental: this API may change in a minor release.
|
||||
type ClientSideCacheConfig = CacheConfig
|
||||
|
||||
const (
|
||||
invalidatePushName = "invalidate"
|
||||
// cscNamespaceSep separates fixed-width/logically-delimited namespace parts
|
||||
// from the command or Redis key.
|
||||
cscNamespaceSep = "\x00"
|
||||
)
|
||||
|
||||
// cscNamespacePrefix scopes a shared cache by database and fixed ACL identity.
|
||||
// Password rotation does not change identity; provider-backed identities are
|
||||
// rejected before attachment.
|
||||
func cscNamespacePrefix(db int, username string) string {
|
||||
return strconv.Itoa(db) + cscNamespaceSep +
|
||||
strconv.Itoa(len(username)) + ":" + username + cscNamespaceSep
|
||||
}
|
||||
|
||||
func cscNamespacedKey(prefix, key string) string {
|
||||
return prefix + key
|
||||
}
|
||||
|
||||
// invalidateHandler propagates RESP3 "invalidate" push notifications into the
|
||||
// shared client-side cache. keyPrefix scopes incoming key names so a shared
|
||||
// cache cannot collide across databases or fixed ACL identities.
|
||||
//
|
||||
// The binding (cache, keyPrefix) is mutable under mu: the owning client's teardown
|
||||
// RELEASES it (cache=nil) instead of unregistering the handler, so the handler
|
||||
// can stay registered protected — application code holding the processor
|
||||
// cannot silently unregister invalidation out from under a live client — while
|
||||
// a successor client on the same processor can still rebind it (see
|
||||
// registerInvalidateHandler).
|
||||
type invalidateHandler struct {
|
||||
mu sync.RWMutex
|
||||
cache Cache
|
||||
keyPrefix string
|
||||
users int
|
||||
}
|
||||
|
||||
// HandlePushNotification decodes ["invalidate", <keys>] notifications. A nil
|
||||
// <keys> payload is emitted on FLUSHDB/FLUSHALL and triggers a full cache flush.
|
||||
func (h *invalidateHandler) HandlePushNotification(
|
||||
_ context.Context, _ push.NotificationHandlerContext, notification []interface{},
|
||||
) error {
|
||||
h.mu.RLock()
|
||||
cache, keyPrefix := h.cache, h.keyPrefix
|
||||
h.mu.RUnlock()
|
||||
if cache == nil || len(notification) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch payload := notification[1].(type) {
|
||||
case nil:
|
||||
cache.Flush()
|
||||
case []interface{}:
|
||||
for _, k := range payload {
|
||||
var name string
|
||||
switch v := k.(type) {
|
||||
case string:
|
||||
name = v
|
||||
case []byte:
|
||||
name = string(v)
|
||||
default:
|
||||
continue
|
||||
}
|
||||
cache.DeleteByRedisKey(cscNamespacedKey(keyPrefix, name))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *invalidateHandler) release() {
|
||||
h.mu.Lock()
|
||||
if h.users > 0 {
|
||||
h.releaseLocked()
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *invalidateHandler) releaseLocked() {
|
||||
h.users--
|
||||
if h.users == 0 {
|
||||
h.cache = nil
|
||||
h.keyPrefix = ""
|
||||
}
|
||||
}
|
||||
|
||||
// sameCache compares Cache interface values without panicking when an
|
||||
// implementation uses a non-comparable value type.
|
||||
func sameCache(a, b Cache) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
typ := reflect.TypeOf(a)
|
||||
return typ == reflect.TypeOf(b) && typ.Comparable() && a == b
|
||||
}
|
||||
|
||||
func isNilCache(cache Cache) bool {
|
||||
if cache == nil {
|
||||
return true
|
||||
}
|
||||
v := reflect.ValueOf(cache)
|
||||
switch v.Kind() {
|
||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
|
||||
return v.IsNil()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// errInvalidateHandlerBound: piggybacking on a handler bound to a live
|
||||
// different cache would leave the new cache uninvalidated.
|
||||
var errInvalidateHandlerBound = errors.New(`csc: a different "invalidate" push handler is already registered`)
|
||||
|
||||
// bindTo binds the handler to (cache, keyPrefix). Success when that is already the
|
||||
// binding (a derived Client.Conn sharing the parent's processor and cache) or
|
||||
// when the handler was released by a previous owner's teardown (rebind);
|
||||
// errInvalidateHandlerBound otherwise.
|
||||
func (h *invalidateHandler) bindTo(cache Cache, keyPrefix string) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
switch {
|
||||
case sameCache(h.cache, cache) && h.keyPrefix == keyPrefix:
|
||||
h.users++
|
||||
return nil
|
||||
case h.cache == nil:
|
||||
h.cache, h.keyPrefix = cache, keyPrefix
|
||||
h.users = 1
|
||||
return nil
|
||||
default:
|
||||
return errInvalidateHandlerBound
|
||||
}
|
||||
}
|
||||
|
||||
// lookupInvalidateHandler returns the processor's CSC invalidate handler, nil
|
||||
// when absent or foreign.
|
||||
func lookupInvalidateHandler(p push.NotificationProcessor) *invalidateHandler {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
h, _ := p.GetHandler(invalidatePushName).(*invalidateHandler)
|
||||
return h
|
||||
}
|
||||
|
||||
func registerInvalidateHandler(p push.NotificationProcessor, cache Cache, keyPrefix string) error {
|
||||
if p == nil || cache == nil {
|
||||
return nil
|
||||
}
|
||||
if existing := p.GetHandler(invalidatePushName); existing != nil {
|
||||
h, ok := existing.(*invalidateHandler)
|
||||
if !ok {
|
||||
return errInvalidateHandlerBound
|
||||
}
|
||||
return h.bindTo(cache, keyPrefix)
|
||||
}
|
||||
// VoidProcessor (RESP2) returns an error here; the caller treats it as
|
||||
// "CSC not available" rather than fatal. Registered PROTECTED: application
|
||||
// code holding the processor must not be able to unregister invalidation
|
||||
// under a live client (that would serve unbounded-stale hits with no
|
||||
// signal); owner teardown releases the BINDING instead of the handler.
|
||||
err := p.RegisterHandler(invalidatePushName, &invalidateHandler{
|
||||
cache: cache,
|
||||
keyPrefix: keyPrefix,
|
||||
users: 1,
|
||||
}, true)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
// Another client can register the same protected handler between GetHandler
|
||||
// and RegisterHandler. Re-read it and accept the compatible binding.
|
||||
if existing := p.GetHandler(invalidatePushName); existing != nil {
|
||||
h, ok := existing.(*invalidateHandler)
|
||||
if !ok {
|
||||
return errInvalidateHandlerBound
|
||||
}
|
||||
return h.bindTo(cache, keyPrefix)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// attachCSC dispatches to the invalidation strategy in
|
||||
// Options.ClientSideCacheStrategy. Safe with a nil cache; on failure c.csc stays
|
||||
// nil and commands fall back to normal round-trips. Adding a strategy: a new
|
||||
// CSCStrategy constant plus cases in Options.init and here.
|
||||
func (c *baseClient) attachCSC(ctx context.Context, cache Cache) {
|
||||
if isNilCache(cache) || c.opt.Protocol != 3 {
|
||||
return
|
||||
}
|
||||
// Credential providers may return a different ACL identity over the
|
||||
// client's lifetime (or per context/connection), while the cache namespace
|
||||
// is fixed when the client is created. Fixed credentials remain safe because
|
||||
// the ACL username is included in the length-delimited namespace below.
|
||||
if c.opt.StreamingCredentialsProvider != nil ||
|
||||
c.opt.CredentialsProviderContext != nil ||
|
||||
c.opt.CredentialsProvider != nil {
|
||||
internal.Logger.Printf(ctx,
|
||||
"redis: client-side caching is disabled with credential providers")
|
||||
return
|
||||
}
|
||||
c.cscKeyPrefix = cscNamespacePrefix(c.opt.DB, c.opt.Username)
|
||||
switch c.opt.ClientSideCacheStrategy {
|
||||
case CSCStrategySharedTracking:
|
||||
c.attachSharedTrackingCSC(ctx, cache)
|
||||
default:
|
||||
// Options.init clamps unknown strategies to SharedTracking; delegate anyway.
|
||||
c.attachSharedTrackingCSC(ctx, cache)
|
||||
}
|
||||
}
|
||||
|
||||
// attachSharedTrackingCSC wires SharedTracking: one shared cache, per-conn CLIENT
|
||||
// TRACKING, a background drainer, and the owning-conn eviction hook. DB-0 only:
|
||||
// tracking is bound to the conn's DB and a runtime SELECT does not re-key it.
|
||||
func (c *baseClient) attachSharedTrackingCSC(ctx context.Context, cache Cache) {
|
||||
if c.opt.DB != 0 {
|
||||
internal.Logger.Printf(ctx,
|
||||
"csc: client-side caching is restricted to DB 0; disabling CSC for client configured with DB=%d. "+
|
||||
"Use one client per DB if you need caching against non-zero databases.", c.opt.DB)
|
||||
return
|
||||
}
|
||||
// A pooler without idle-conn draining (e.g. Client.Conn's StickyConnPool)
|
||||
// can't apply buffered invalidations, so stay uncached.
|
||||
if _, ok := c.connPool.(idleConnDrainer); !ok {
|
||||
return
|
||||
}
|
||||
// The lifecycle hook serializes cache publication with connection removal
|
||||
// and socket replacement. Without it, a reply can become visible after its
|
||||
// tracking coverage is gone.
|
||||
reg, ok := c.connPool.(poolHookSupport)
|
||||
if !ok || !reg.SupportsPoolHooks() {
|
||||
return
|
||||
}
|
||||
if err := registerInvalidateHandler(c.pushProcessor, cache, c.cscKeyPrefix); err != nil {
|
||||
internal.Logger.Printf(ctx, "csc: failed to register invalidate handler: %v", err)
|
||||
return
|
||||
}
|
||||
c.csc = cache
|
||||
c.registerConnEvictHook(cache, reg)
|
||||
c.startBackgroundDrainer()
|
||||
}
|
||||
|
||||
// cscHook returns the shared evict-on-remove hook, nil when CSC is off.
|
||||
func (c *baseClient) cscHook() *cscEvictOnRemoveHook {
|
||||
h, _ := c.cscPoolHook.(*cscEvictOnRemoveHook)
|
||||
return h
|
||||
}
|
||||
|
||||
// cscInstallConnCloseHook evicts cn's owned entries on any close — including the
|
||||
// ConnMaxLifetime/idle retirement path (CloseConn) that bypasses the OnRemove
|
||||
// hook — so entries don't outlive the server tracking dropped at close. Uses the
|
||||
// onCscClose slot so it doesn't clobber streaming-credentials cleanup.
|
||||
func (c *baseClient) cscInstallConnCloseHook(cn *pool.Conn) {
|
||||
cn.SetOnCscClose(func() error {
|
||||
c.cscOnConnClose(cn.GetID())
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// cscInstallConnReinitHook invalidates the old socket's cache coverage before
|
||||
// SetNetConnAndInitConn replaces it. The later init can then safely enable
|
||||
// tracking for the new socket without a post-swap publication window.
|
||||
func (c *baseClient) cscInstallConnReinitHook(cn *pool.Conn) {
|
||||
cn.SetOnCscReinit(func() {
|
||||
c.cscEvictOwnedEntries(cn.GetID())
|
||||
})
|
||||
}
|
||||
|
||||
// cscOnConnClose evicts a closing conn's entries: via the shared hook (which
|
||||
// records the removed-ring, closing the close-before-fulfill race), else scoped
|
||||
// EvictByConn on the owning cache.
|
||||
func (c *baseClient) cscOnConnClose(connID uint64) {
|
||||
if h := c.cscHook(); h != nil {
|
||||
h.markRemoved(connID)
|
||||
return
|
||||
}
|
||||
if c.csc != nil {
|
||||
c.csc.EvictByConn(connID)
|
||||
}
|
||||
}
|
||||
|
||||
// poolHookSupport is the pool capability SharedTracking needs to serialize
|
||||
// cache publication with connection removal and reinitialization.
|
||||
type poolHookSupport interface {
|
||||
AddPoolHook(hook pool.PoolHook)
|
||||
RemovePoolHook(hook pool.PoolHook)
|
||||
SupportsPoolHooks() bool
|
||||
}
|
||||
|
||||
// cscEvictOnRemoveHook evicts a connection's owned entries when the pool removes
|
||||
// it (the server stops delivering their invalidations — Window 2), and tracks
|
||||
// per-conn init generations so fulfillCached can catch a value whose owning
|
||||
// conn was removed or re-initialized mid-fetch.
|
||||
type cscEvictOnRemoveHook struct {
|
||||
evictor Cache
|
||||
|
||||
mu sync.Mutex
|
||||
// initGen counts a live conn's socket (re)initializations: bumped by
|
||||
// cscEvictOwnedEntries before its eviction (first init included, so every
|
||||
// serving conn has gen >= 1), deleted on removal/close. fulfillCached
|
||||
// compares it with the generation captured at reply time.
|
||||
initGen map[uint64]uint64
|
||||
}
|
||||
|
||||
func (h *cscEvictOnRemoveHook) OnGet(_ context.Context, _ *pool.Conn, _ bool) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (h *cscEvictOnRemoveHook) OnPut(_ context.Context, _ *pool.Conn) (shouldPool, shouldRemove bool, err error) {
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
func (h *cscEvictOnRemoveHook) OnRemove(_ context.Context, cn *pool.Conn, _ error) {
|
||||
if cn == nil {
|
||||
return
|
||||
}
|
||||
h.markRemoved(cn.GetID())
|
||||
}
|
||||
|
||||
// markRemoved forgets connID's generation, then evicts. Forgetting before
|
||||
// evicting lets a racing fulfillCached see the change (a served conn's captured
|
||||
// generation is >= 1, an absent entry reads 0) and drop an entry created after
|
||||
// the eviction — closing the close-before-fulfill race.
|
||||
func (h *cscEvictOnRemoveHook) markRemoved(connID uint64) {
|
||||
h.forgetConn(connID)
|
||||
h.evictor.EvictByConn(connID)
|
||||
}
|
||||
|
||||
// bumpInitGen advances connID's coverage generation. On reinit it is called by
|
||||
// the pre-swap hook, before the old socket and its server-side tracking table
|
||||
// are replaced.
|
||||
func (h *cscEvictOnRemoveHook) bumpInitGen(connID uint64) {
|
||||
h.mu.Lock()
|
||||
if h.initGen == nil {
|
||||
h.initGen = make(map[uint64]uint64)
|
||||
}
|
||||
h.initGen[connID]++
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// invalidateConnCoverage revokes all cache coverage associated with connID.
|
||||
// Bumping before eviction also rejects an in-flight fetch that completed on the
|
||||
// connection just before it left the parent's invalidation drainer.
|
||||
func (h *cscEvictOnRemoveHook) invalidateConnCoverage(connID uint64) {
|
||||
h.bumpInitGen(connID)
|
||||
h.evictor.EvictByConn(connID)
|
||||
}
|
||||
|
||||
// initGenOf returns connID's current init generation (0 if never bumped).
|
||||
func (h *cscEvictOnRemoveHook) initGenOf(connID uint64) uint64 {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.initGen[connID]
|
||||
}
|
||||
|
||||
// forgetConn drops connID's init-generation entry: the conn was removed/closed,
|
||||
// or its init failed before ever serving (the pubsub path would otherwise leak
|
||||
// the entry — no OnRemove hook, close hook not yet installed).
|
||||
func (h *cscEvictOnRemoveHook) forgetConn(connID uint64) {
|
||||
h.mu.Lock()
|
||||
delete(h.initGen, connID)
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// fulfillOwnedIfCovered linearizes the final coverage check with connection
|
||||
// removal/re-init generation changes. Holding h.mu through FulfillOwned means
|
||||
// either the old generation is rejected before the placeholder becomes valid,
|
||||
// or publication wins first and the subsequent lifecycle path evicts it before
|
||||
// closing/replacing the tracked socket.
|
||||
func (h *cscEvictOnRemoveHook) fulfillOwnedIfCovered(
|
||||
cacheKey string,
|
||||
token, ownerConnID, capturedGen uint64,
|
||||
value []byte,
|
||||
) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.initGen[ownerConnID] != capturedGen {
|
||||
return false
|
||||
}
|
||||
return h.evictor.FulfillOwned(cacheKey, token, ownerConnID, value)
|
||||
}
|
||||
|
||||
// invalidateAllCoverage revokes every connection generation known to this
|
||||
// client's pool and evicts the entries those connections own. Incrementing
|
||||
// instead of deleting keeps in-flight fetches that captured an old generation
|
||||
// from publishing after a drainer stops.
|
||||
func (h *cscEvictOnRemoveHook) invalidateAllCoverage() {
|
||||
h.mu.Lock()
|
||||
connIDs := make([]uint64, 0, len(h.initGen))
|
||||
for connID := range h.initGen {
|
||||
h.initGen[connID]++
|
||||
connIDs = append(connIDs, connID)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
|
||||
for _, connID := range connIDs {
|
||||
h.evictor.EvictByConn(connID)
|
||||
}
|
||||
}
|
||||
|
||||
// registerConnEvictHook wires the required OnRemove eviction hook.
|
||||
func (c *baseClient) registerConnEvictHook(cache Cache, reg poolHookSupport) {
|
||||
h := &cscEvictOnRemoveHook{evictor: cache, initGen: make(map[uint64]uint64)}
|
||||
reg.AddPoolHook(h)
|
||||
c.cscPoolHook = h
|
||||
}
|
||||
|
||||
// cscEvictOwnedEntries evicts connID's entries on first init or immediately
|
||||
// before a reinit/handoff replaces the socket and its tracking table. It
|
||||
// prefers the shared hook (so Conn/Tx, which carry it but have a nil csc, still
|
||||
// evict from the parent cache). Scoped only — no removed-ring (the conn keeps
|
||||
// serving, and the ring never ages out); the fulfill-vs-re-init race is closed
|
||||
// by the init-generation bump instead. No custom-cache flush (this also runs on
|
||||
// first init).
|
||||
func (c *baseClient) cscEvictOwnedEntries(connID uint64) {
|
||||
if h := c.cscHook(); h != nil {
|
||||
h.invalidateConnCoverage(connID)
|
||||
return
|
||||
}
|
||||
if c.csc == nil {
|
||||
return
|
||||
}
|
||||
c.csc.EvictByConn(connID)
|
||||
}
|
||||
|
||||
// newStickyConnPool creates a derived sticky pool and revokes the claimed
|
||||
// connection's parent-cache ownership before it becomes unreachable to the
|
||||
// parent's idle-connection drainer.
|
||||
func (c *baseClient) newStickyConnPool() *pool.StickyConnPool {
|
||||
sticky := pool.NewStickyConnPool(c.connPool)
|
||||
if h := c.cscHook(); h != nil {
|
||||
sticky.SetOnFirstConn(func(cn *pool.Conn) {
|
||||
if cn != nil {
|
||||
h.invalidateConnCoverage(cn.GetID())
|
||||
}
|
||||
})
|
||||
}
|
||||
return sticky
|
||||
}
|
||||
|
||||
// cscFetchCapture receives, from the successful attempt's reply read — while
|
||||
// the serving connection is still held — everything the CSC fetch path needs to
|
||||
// attribute the cached entry: the raw RESP reply, the conn id, and the conn's
|
||||
// CSC init generation. The generation must be captured before the conn is
|
||||
// released: a handoff queued at Put can re-init the socket (bumping the
|
||||
// generation) before fulfillCached runs.
|
||||
type cscFetchCapture struct {
|
||||
raw []byte
|
||||
connID uint64
|
||||
initGen uint64
|
||||
}
|
||||
|
||||
// cscConnInitGen returns connID's CSC init generation, captured by _process at
|
||||
// reply time (while the conn is still held) and compared by fulfillCached via
|
||||
// fulfillOwnedIfCovered. Zero without an active evict-on-remove hook.
|
||||
func (c *baseClient) cscConnInitGen(connID uint64) uint64 {
|
||||
if h := c.cscHook(); h != nil {
|
||||
return h.initGenOf(connID)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// cscForgetConn drops connID's init-generation entry when initialization does
|
||||
// not establish tracked coverage, either because init failed or tracking was
|
||||
// rejected and CSC was disabled.
|
||||
func (c *baseClient) cscForgetConn(connID uint64) {
|
||||
if h := c.cscHook(); h != nil {
|
||||
h.forgetConn(connID)
|
||||
}
|
||||
}
|
||||
|
||||
// errClientTrackingWithCSC rejects CLIENT TRACKING on clients with built-in CSC
|
||||
// (see the guards in baseClient.process and generalProcessPipeline). The raw
|
||||
// escape hatches — Do(ctx, "client", "tracking", ...) with string or []byte
|
||||
// args, and pipelines — are also caught: the guard matches on the command's
|
||||
// leading args, not the typed method.
|
||||
var errClientTrackingWithCSC = errors.New(
|
||||
"redis: CLIENT TRACKING is not allowed when client-side caching is enabled")
|
||||
|
||||
// errSelectWithCSC rejects runtime SELECT on clients with built-in CSC. Cache
|
||||
// keys use Options.DB, while SELECT mutates only the chosen pool connection.
|
||||
var errSelectWithCSC = errors.New(
|
||||
"redis: SELECT is not allowed when client-side caching is enabled")
|
||||
|
||||
// errAuthWithCSC rejects runtime authentication because it can change one
|
||||
// connection's ACL identity without changing the client's fixed cache namespace.
|
||||
var errAuthWithCSC = errors.New(
|
||||
"redis: AUTH is not allowed when client-side caching is enabled")
|
||||
|
||||
// errHelloWithCSC rejects HELLO with arguments because it can switch a tracked
|
||||
// connection out of RESP3 (and can also change authentication).
|
||||
var errHelloWithCSC = errors.New(
|
||||
"redis: HELLO with arguments is not allowed when client-side caching is enabled")
|
||||
|
||||
// errResetWithCSC rejects RESET because it disables tracking and switches the
|
||||
// connection to RESP2.
|
||||
var errResetWithCSC = errors.New(
|
||||
"redis: RESET is not allowed when client-side caching is enabled")
|
||||
|
||||
// errSubscribeWithCSC rejects raw subscriptions on the ordinary pool. The
|
||||
// typed Subscribe methods use dedicated PubSub connections and remain allowed.
|
||||
var errSubscribeWithCSC = errors.New(
|
||||
"redis: SUBSCRIBE is not allowed on pooled connections when client-side caching is enabled")
|
||||
|
||||
// cscCommandError rejects commands that can make a pooled connection's state
|
||||
// diverge from the assumptions used by CSC.
|
||||
func (c *baseClient) cscCommandError(cmd Cmder) error {
|
||||
// The successful attachment signal is shared with derived clients.
|
||||
// initConn's internal command wrapper is exempt during library setup.
|
||||
if !c.cscTrackingRequested() || c.allowClientTracking {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case isClientTrackingCmd(cmd):
|
||||
return errClientTrackingWithCSC
|
||||
case isSelectCmd(cmd):
|
||||
return errSelectWithCSC
|
||||
case isAuthCmd(cmd):
|
||||
return errAuthWithCSC
|
||||
case isProtocolChangingHelloCmd(cmd):
|
||||
return errHelloWithCSC
|
||||
case isResetCmd(cmd):
|
||||
return errResetWithCSC
|
||||
case isSubscribeCmd(cmd):
|
||||
return errSubscribeWithCSC
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// cscDrainHandle owns the drainer lifecycle and serializes client teardown.
|
||||
// stop signals shutdown; done is closed on exit so Close can join.
|
||||
type cscDrainHandle struct {
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
stopOnce sync.Once
|
||||
teardownOnce sync.Once
|
||||
handlerCloseOnce sync.Once
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
invalidateHandler *invalidateHandler
|
||||
}
|
||||
|
||||
// signalStop closes stop at most once (so Close and the AddCleanup safety net
|
||||
// can't double-close) and does not join — a GC cleanup must not block.
|
||||
func (h *cscDrainHandle) signalStop() {
|
||||
h.stopOnce.Do(func() { close(h.stop) })
|
||||
}
|
||||
|
||||
// cscHandlerClient is exposed only through the background drainer's handler
|
||||
// context. Close must return before the handler does, otherwise it would wait
|
||||
// for the drainer goroutine that is currently invoking the handler.
|
||||
type cscHandlerClient struct {
|
||||
*baseClient
|
||||
}
|
||||
|
||||
func (c cscHandlerClient) Close() error {
|
||||
h := c.cscDrainHandle
|
||||
if h == nil {
|
||||
return c.baseClient.Close()
|
||||
}
|
||||
h.handlerCloseOnce.Do(func() {
|
||||
// Close has logically started: stop cache hits immediately and let the
|
||||
// drainer exit as soon as this handler returns.
|
||||
if c.cscActive != nil {
|
||||
c.cscActive.Store(false)
|
||||
}
|
||||
h.signalStop()
|
||||
go func() {
|
||||
if err := c.baseClient.Close(); err != nil {
|
||||
internal.Logger.Printf(context.Background(), "csc: deferred client close failed: %v", err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// cscMinDrainInterval floors a user-supplied DrainInterval: sub-millisecond
|
||||
// timers are unreliable (https://github.com/golang/go/issues/53824).
|
||||
const cscMinDrainInterval = time.Millisecond
|
||||
|
||||
// cscDrainInterval returns DrainInterval clamped to cscMinDrainInterval, or the
|
||||
// default (cscDrainSkipWindow) when unset.
|
||||
func (c *baseClient) cscDrainInterval() time.Duration {
|
||||
if cfg := c.opt.ClientSideCacheConfig; cfg != nil && cfg.DrainInterval > 0 {
|
||||
if cfg.DrainInterval < cscMinDrainInterval {
|
||||
return cscMinDrainInterval
|
||||
}
|
||||
return cfg.DrainInterval
|
||||
}
|
||||
return cscDrainSkipWindow
|
||||
}
|
||||
|
||||
// idleConnDrainer is the pooler capability the drainer needs (*pool.ConnPool has
|
||||
// it). attachSharedTrackingCSC leaves a pooler without it uncached, rather than
|
||||
// serve entries nothing would invalidate.
|
||||
type idleConnDrainer interface {
|
||||
DrainIdleConns(ctx context.Context, st *pool.DrainState, fn func(cn *pool.Conn) error)
|
||||
}
|
||||
|
||||
// startBackgroundDrainer launches the per-client invalidation drainer: each tick
|
||||
// runs one pool.DrainIdleConns pass, draining idle conns' buffered push frames.
|
||||
// No-op for poolers that don't implement idleConnDrainer.
|
||||
func (c *baseClient) startBackgroundDrainer() {
|
||||
cp, ok := c.connPool.(idleConnDrainer)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if c.cscDrainHandle != nil {
|
||||
return // already running (startBackgroundDrainer runs once, in NewClient)
|
||||
}
|
||||
h := &cscDrainHandle{
|
||||
stop: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
invalidateHandler: lookupInvalidateHandler(c.pushProcessor),
|
||||
}
|
||||
c.cscDrainHandle = h
|
||||
active := &atomic.Bool{}
|
||||
active.Store(true)
|
||||
c.cscActive = active
|
||||
interval := c.cscDrainInterval()
|
||||
// Custom-processor drain errors are connection-fatal (drainPushNotifications),
|
||||
// so a PERSISTENTLY failing custom processor would turn every tick into a
|
||||
// conn removal + redial — a sustained dial storm. Damping: after
|
||||
// cscDrainCustomErrCap consecutive fatal custom-processor drains, disable
|
||||
// CSC serving and stop the drainer (with one log line) instead of churning.
|
||||
// Built-in processor errors are real conn desyncs and are never damped.
|
||||
_, builtinProc := c.pushProcessor.(*push.Processor)
|
||||
go func() {
|
||||
defer func() {
|
||||
active.Store(false)
|
||||
if c.cscPoolHook != nil {
|
||||
if reg, ok := c.connPool.(poolHookSupport); ok {
|
||||
reg.RemovePoolHook(c.cscPoolHook)
|
||||
}
|
||||
}
|
||||
if hook := c.cscHook(); hook != nil {
|
||||
hook.invalidateAllCoverage()
|
||||
}
|
||||
if h.invalidateHandler != nil {
|
||||
h.invalidateHandler.release()
|
||||
}
|
||||
close(h.done)
|
||||
}()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
// st persists round/visited across ticks; single-goroutine, no lock.
|
||||
var st pool.DrainState
|
||||
consecFatal := 0
|
||||
drain := func(cn *pool.Conn) error {
|
||||
processorSucceeded, err := c.drainPushNotifications(cn)
|
||||
switch {
|
||||
case err != nil:
|
||||
consecFatal++
|
||||
case processorSucceeded:
|
||||
// A successful processor invocation resets consecutive
|
||||
// failures. A conn skipped without invoking the processor —
|
||||
// including a clean replacement after a fatal drain — does
|
||||
// not reset the counter.
|
||||
consecFatal = 0
|
||||
}
|
||||
return err
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-h.stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
if !active.Load() {
|
||||
return
|
||||
}
|
||||
// ctx bounds the whole pass; the drain read has its own hard deadline.
|
||||
cycleCtx, cancel := context.WithTimeout(context.Background(), interval/2)
|
||||
cp.DrainIdleConns(cycleCtx, &st, drain)
|
||||
cancel()
|
||||
if !builtinProc && consecFatal >= cscDrainCustomErrCap {
|
||||
internal.Logger.Printf(context.Background(),
|
||||
"csc: disabling client-side caching: the custom push notification processor failed %d consecutive drains "+
|
||||
"(each failure removes a connection because the reader may be mid-frame); "+
|
||||
"caching cannot be kept fresh safely with this processor", consecFatal)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// disableCSCServing atomically stops cache hits and revokes all tracked
|
||||
// connection coverage. The owner drainer observes the shared active flag on its
|
||||
// next tick, including when a derived Conn or Tx discovered the incompatibility.
|
||||
func (c *baseClient) disableCSCServing(ctx context.Context, reason string) {
|
||||
active := c.cscActive
|
||||
if active == nil || !active.CompareAndSwap(true, false) {
|
||||
return
|
||||
}
|
||||
if hook := c.cscHook(); hook != nil {
|
||||
hook.invalidateAllCoverage()
|
||||
}
|
||||
internal.Logger.Printf(ctx, "csc: disabling client-side caching: %s", reason)
|
||||
}
|
||||
|
||||
// stopBackgroundDrainer joins the drainer goroutine and flushes an owned cache.
|
||||
// The drainer's exit path releases its handler binding and pool hook, including
|
||||
// when it stops itself. Owner-only: clones have no handle and return early.
|
||||
// The fields are never cleared here — fulfillCached reads cscPoolHook on the hot
|
||||
// path, so niling under a concurrent Close would race; teardownOnce makes repeat
|
||||
// Close idempotent instead.
|
||||
func (c *baseClient) stopBackgroundDrainer() {
|
||||
h := c.cscDrainHandle
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
h.teardownOnce.Do(func() {
|
||||
// Stop serving cache hits on any clone before the drainer is gone.
|
||||
if c.cscActive != nil {
|
||||
c.cscActive.Store(false)
|
||||
}
|
||||
h.signalStop()
|
||||
<-h.done
|
||||
// The drainer's exit defer revoked and evicted this pool's coverage
|
||||
// before closing done, including for injected caches shared elsewhere.
|
||||
if c.cscOwnsCache && c.csc != nil {
|
||||
c.csc.Flush()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// applyCachedReply populates cmd from a previously captured raw RESP reply by
|
||||
// replaying it through the command's own readReply.
|
||||
func applyCachedReply(cmd Cmder, raw []byte) error {
|
||||
return cmd.readReply(proto.NewReaderSize(bytes.NewReader(raw), len(raw)+1))
|
||||
}
|
||||
|
||||
// isCacheableReplyResult reports whether a fully read Redis reply can be
|
||||
// cached. redis.Nil is a normal negative lookup, not a transport/protocol
|
||||
// failure; tracking will invalidate it if the key is later created.
|
||||
func isCacheableReplyResult(err error) bool {
|
||||
return err == nil || err == Nil
|
||||
}
|
||||
|
||||
// cscDrainSkipWindow is the default SharedTracking drain period (overridable via
|
||||
// ClientSideCacheConfig.DrainInterval). A buffered invalidation is picked up within
|
||||
// roughly one round; MaxStaleness, when configured, is the hard time-based backstop.
|
||||
const cscDrainSkipWindow = 5 * time.Millisecond
|
||||
|
||||
// cscDrainHardReadCap is the hard socket read deadline the drainer applies via
|
||||
// Conn.WithReaderHardDeadline. It bounds only a rare partial-frame mid-read. A
|
||||
// var (not const) so the tuning harness can sweep it.
|
||||
var cscDrainHardReadCap = 50 * time.Millisecond
|
||||
|
||||
// cscDrainProbeReadCap bounds the non-consuming one-byte probe used only when
|
||||
// an opaque transport may hold data that the socket readiness check cannot see.
|
||||
const cscDrainProbeReadCap = 50 * time.Microsecond
|
||||
|
||||
// cscDrainCustomErrCap is the number of CONSECUTIVE fatal custom-processor
|
||||
// drain errors after which the drainer disables CSC instead of removing (and
|
||||
// redialing) a connection per tick indefinitely.
|
||||
const cscDrainCustomErrCap = 8
|
||||
|
||||
// processCached runs the Get-Reserve-Fulfill lifecycle for a cacheable command.
|
||||
// Only invoked after process has verified that CSC is active and cmd is
|
||||
// eligible.
|
||||
func (c *baseClient) processCached(ctx context.Context, cmd Cmder, state *processState) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Once the drainer has stopped (owner Close, or the owner dropped without
|
||||
// Close), no invalidations flow — a surviving clone must not serve stale hits.
|
||||
if a := c.cscActive; a != nil && !a.Load() {
|
||||
return c.processWithRetry(ctx, cmd, nil, state)
|
||||
}
|
||||
|
||||
rawKey, ok := buildCacheKey(cmd)
|
||||
if !ok {
|
||||
return c.processWithRetry(ctx, cmd, nil, state)
|
||||
}
|
||||
|
||||
redisKeys := extractRedisKeys(cmd)
|
||||
if len(redisKeys) == 0 {
|
||||
// Without a key list we cannot react to invalidations for this command.
|
||||
return c.processWithRetry(ctx, cmd, nil, state)
|
||||
}
|
||||
|
||||
keyPrefix := c.cscKeyPrefix
|
||||
if keyPrefix == "" {
|
||||
// A successfully attached client always has a namespace. Fail closed if
|
||||
// an incomplete custom baseClient reaches this path.
|
||||
return c.processWithRetry(ctx, cmd, nil, state)
|
||||
}
|
||||
key := cscNamespacedKey(keyPrefix, rawKey)
|
||||
nsRedisKeys := make([]string, len(redisKeys))
|
||||
for i, k := range redisKeys {
|
||||
nsRedisKeys[i] = cscNamespacedKey(keyPrefix, k)
|
||||
}
|
||||
|
||||
// Serve hits straight from the cache.
|
||||
if data, ok := c.csc.Get(ctx, key); ok {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := applyCachedReply(cmd, data); isCacheableReplyResult(err) {
|
||||
return err
|
||||
}
|
||||
c.csc.DeleteByCacheKey(key)
|
||||
}
|
||||
|
||||
token, shouldFetch := c.csc.Reserve(key, nsRedisKeys)
|
||||
if !shouldFetch {
|
||||
// Another goroutine is fetching; Get below waits until it completes.
|
||||
if data, ok := c.csc.Get(ctx, key); ok {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := applyCachedReply(cmd, data); isCacheableReplyResult(err) {
|
||||
return err
|
||||
}
|
||||
c.csc.DeleteByCacheKey(key)
|
||||
}
|
||||
// Original fetcher cancelled or its value was invalidated; try to take
|
||||
// over so later waiters still benefit from the cache.
|
||||
token, shouldFetch = c.csc.Reserve(key, nsRedisKeys)
|
||||
}
|
||||
|
||||
var fc cscFetchCapture
|
||||
var capture *cscFetchCapture
|
||||
if shouldFetch {
|
||||
capture = &fc
|
||||
// Release the placeholder if processWithRetry panics; Cancel on a
|
||||
// stale token is a no-op.
|
||||
defer func() {
|
||||
if capture != nil {
|
||||
c.csc.Cancel(key, token)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
err := c.processWithRetry(ctx, cmd, capture, state)
|
||||
|
||||
if shouldFetch {
|
||||
capture = nil // disarm the deferred Cancel
|
||||
if isCacheableReplyResult(err) {
|
||||
c.fulfillCached(key, token, &fc)
|
||||
} else {
|
||||
c.csc.Cancel(key, token)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// fulfillCached stores a fetched value, attributing it to its serving conn when
|
||||
// an evict-on-remove hook is active so EvictByConn can drop it if that conn is
|
||||
// removed. It also closes the attribute-vs-coverage races: the conn is released
|
||||
// before this runs, so its OnRemove eviction — or a handoff re-init's scoped
|
||||
// eviction — may fire before the entry exists. Publication is serialized with
|
||||
// the hook's init-generation changes, so a reply whose invalidation coverage
|
||||
// was already lost never becomes visible and never wakes waiters with stale
|
||||
// data.
|
||||
func (c *baseClient) fulfillCached(key string, token uint64, fc *cscFetchCapture) bool {
|
||||
if active := c.cscActive; active != nil && !active.Load() {
|
||||
c.csc.Cancel(key, token)
|
||||
return false
|
||||
}
|
||||
if hook := c.cscHook(); hook != nil {
|
||||
if fc.connID == 0 {
|
||||
// Invariant: an active hook always gets a real conn id (>=1). A zero id
|
||||
// would leave the entry unattributed and un-evictable, so fail closed.
|
||||
c.csc.Cancel(key, token)
|
||||
return false
|
||||
}
|
||||
if !hook.fulfillOwnedIfCovered(key, token, fc.connID, fc.initGen, fc.raw) {
|
||||
// A coverage mismatch leaves the reservation IN_PROGRESS because
|
||||
// FulfillOwned was deliberately skipped. Cancel wakes its waiters
|
||||
// as misses so one can safely refetch on a covered connection.
|
||||
c.csc.Cancel(key, token)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return c.csc.FulfillOwned(key, token, 0, fc.raw)
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package redis
|
||||
|
||||
// CSCStats reports cumulative client-side cache activity and current
|
||||
// residency.
|
||||
//
|
||||
// Experimental: this API may change in a minor release.
|
||||
type CSCStats struct {
|
||||
Hits uint64
|
||||
Misses uint64
|
||||
Entries int
|
||||
MemoryUsageBytes int64
|
||||
}
|
||||
|
||||
// cacheStatsReporter is an optional interface a Cache implementation may
|
||||
// satisfy to expose statistics. The built-in LocalCache does; user
|
||||
// implementations are not required to.
|
||||
type cacheStatsReporter interface {
|
||||
Stats() CSCStats
|
||||
}
|
||||
|
||||
// CSCStats returns statistics for this client's client-side cache, read from
|
||||
// the shared cache when its implementation exposes them (the built-in
|
||||
// LocalCache does).
|
||||
//
|
||||
// It returns a zero value when CSC is not configured or stats are unavailable.
|
||||
//
|
||||
// Experimental: this API may change in a minor release.
|
||||
func (c *Client) CSCStats() CSCStats {
|
||||
if c == nil || c.baseClient.csc == nil {
|
||||
return CSCStats{}
|
||||
}
|
||||
if r, ok := c.baseClient.csc.(cacheStatsReporter); ok {
|
||||
return r.Stats()
|
||||
}
|
||||
return CSCStats{}
|
||||
}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
---
|
||||
|
||||
x-default-image: &default-image ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.8.0}
|
||||
x-default-image: &default-image ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.10.0}
|
||||
|
||||
services:
|
||||
redis:
|
||||
|
||||
+3
@@ -168,6 +168,9 @@ func shouldRetry(err error, retryTimeout bool) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Other server errors are not retried. This includes the logical
|
||||
// -SEARCH_TIMEOUT (search-on-timeout fail): retrying would just repeat the
|
||||
// same expensive query.
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
+6
@@ -129,6 +129,9 @@ func (c cmdable) ExpireAt(ctx context.Context, key string, tm time.Time) *BoolCm
|
||||
return cmd
|
||||
}
|
||||
|
||||
// ExpireTime returns the absolute expiration time of key as a Unix timestamp
|
||||
// encoded in *DurationCmd (seconds since the epoch), not a remaining TTL.
|
||||
// Convert with: time.Unix(int64(d/time.Second), 0). Use TTL/PTTL for remaining TTL.
|
||||
func (c cmdable) ExpireTime(ctx context.Context, key string) *DurationCmd {
|
||||
cmd := NewDurationCmd(ctx, time.Second, "expiretime", key)
|
||||
_ = c(ctx, cmd)
|
||||
@@ -209,6 +212,9 @@ func (c cmdable) PExpireAt(ctx context.Context, key string, tm time.Time) *BoolC
|
||||
return cmd
|
||||
}
|
||||
|
||||
// PExpireTime returns the absolute expiration time of key as a Unix timestamp
|
||||
// encoded in *DurationCmd (milliseconds since the epoch), not a remaining TTL.
|
||||
// Convert with: time.UnixMilli(int64(d/time.Millisecond)). Use TTL/PTTL for remaining TTL.
|
||||
func (c cmdable) PExpireTime(ctx context.Context, key string) *DurationCmd {
|
||||
cmd := NewDurationCmd(ctx, time.Millisecond, "pexpiretime", key)
|
||||
_ = c(ctx, cmd)
|
||||
|
||||
+5
@@ -44,6 +44,11 @@ type HashCmdable interface {
|
||||
HPExpireTime(ctx context.Context, key string, fields ...string) *IntSliceCmd
|
||||
HTTL(ctx context.Context, key string, fields ...string) *IntSliceCmd
|
||||
HPTTL(ctx context.Context, key string, fields ...string) *IntSliceCmd
|
||||
// note: the HIMPORT API is experimental and may be subject to change.
|
||||
HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd
|
||||
HImportSet(ctx context.Context, key, fieldsetName string, values ...interface{}) *StatusCmd
|
||||
HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd
|
||||
HImportDiscardAll(ctx context.Context) *IntCmd
|
||||
}
|
||||
|
||||
func (c cmdable) HDel(ctx context.Context, key string, fields ...string) *IntCmd {
|
||||
|
||||
+473
@@ -0,0 +1,473 @@
|
||||
package redis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/redis/go-redis/v9/internal"
|
||||
"github.com/redis/go-redis/v9/internal/pool"
|
||||
"github.com/redis/go-redis/v9/internal/proto"
|
||||
)
|
||||
|
||||
// himportFieldset is the client-side record of a fieldset registered with
|
||||
// HImportPrepare.
|
||||
type himportFieldset struct {
|
||||
fields []string
|
||||
version uint64
|
||||
}
|
||||
|
||||
// himportRegistry remembers fieldsets registered through a client so HIMPORT
|
||||
// SET can lazily prepare them on whichever pooled connection it executes.
|
||||
// Versions increase monotonically and start at 1; re-registering a name under
|
||||
// a new version invalidates every connection's prepared flag for it, so a
|
||||
// replaced fieldset is re-prepared before its next use.
|
||||
//
|
||||
// Discards propagate lazily as well: a discarded name is kept as a tombstone
|
||||
// and the discard-all counter as an epoch, and connections whose sessions
|
||||
// still hold discarded fieldsets replay HIMPORT DISCARD/DISCARDALL before
|
||||
// their next HIMPORT command (see baseClient.himportInjectedCmds).
|
||||
type himportRegistry struct {
|
||||
mu sync.RWMutex
|
||||
nextVersion uint64
|
||||
fieldsets map[string]himportFieldset
|
||||
// tombstones holds names discarded through this client whose server-side
|
||||
// copies may survive on pooled connections that prepared them. An entry
|
||||
// is removed when the name is registered again (the new version replaces
|
||||
// the fieldset on the server, so no discard is needed) or by discardAll.
|
||||
// Known limitation: a workload discarding many uniquely-named fieldsets
|
||||
// grows this map for the client's lifetime and pays an O(tombstones)
|
||||
// snapshot per HIMPORT round trip; HImportDiscardAll resets it.
|
||||
tombstones map[string]struct{}
|
||||
// discardAllEpoch increments on every successful HImportDiscardAll.
|
||||
discardAllEpoch uint64
|
||||
}
|
||||
|
||||
func newHImportRegistry() *himportRegistry {
|
||||
return &himportRegistry{}
|
||||
}
|
||||
|
||||
// register stores the fieldset and returns its new version together with the
|
||||
// current discard-all epoch.
|
||||
func (r *himportRegistry) register(name string, fields []string) (version, epoch uint64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.fieldsets == nil {
|
||||
r.fieldsets = make(map[string]himportFieldset)
|
||||
}
|
||||
delete(r.tombstones, name)
|
||||
r.nextVersion++
|
||||
r.fieldsets[name] = himportFieldset{
|
||||
fields: append([]string(nil), fields...),
|
||||
version: r.nextVersion,
|
||||
}
|
||||
return r.nextVersion, r.discardAllEpoch
|
||||
}
|
||||
|
||||
func (r *himportRegistry) lookup(name string) (himportFieldset, bool) {
|
||||
if r == nil {
|
||||
return himportFieldset{}, false
|
||||
}
|
||||
r.mu.RLock()
|
||||
fs, ok := r.fieldsets[name]
|
||||
r.mu.RUnlock()
|
||||
return fs, ok
|
||||
}
|
||||
|
||||
// discard removes the fieldset and leaves a tombstone so connections whose
|
||||
// sessions still hold it replay the DISCARD before their next HIMPORT
|
||||
// command. It reports whether the fieldset was registered.
|
||||
func (r *himportRegistry) discard(name string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, ok := r.fieldsets[name]; !ok {
|
||||
return false
|
||||
}
|
||||
delete(r.fieldsets, name)
|
||||
if r.tombstones == nil {
|
||||
r.tombstones = make(map[string]struct{})
|
||||
}
|
||||
r.tombstones[name] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
// discardAll drops every fieldset and tombstone and moves to a new epoch;
|
||||
// connections prepared under an older epoch replay HIMPORT DISCARDALL before
|
||||
// their next HIMPORT command. It returns the new epoch and the number of
|
||||
// fieldsets that were registered.
|
||||
func (r *himportRegistry) discardAll() (epoch uint64, removed int) {
|
||||
r.mu.Lock()
|
||||
removed = len(r.fieldsets)
|
||||
r.fieldsets = nil
|
||||
r.tombstones = nil
|
||||
r.discardAllEpoch++
|
||||
epoch = r.discardAllEpoch
|
||||
r.mu.Unlock()
|
||||
return epoch, removed
|
||||
}
|
||||
|
||||
// discardVersion withdraws a registration whose fan-out PREPARE was rejected
|
||||
// by a server — but only while the entry is still at that version, so a
|
||||
// concurrent re-registration is not clobbered. A tombstone is left: the
|
||||
// fan-out may have succeeded on some masters before another rejected it
|
||||
// (per-node ACLs, rolling upgrades), and those sessions hold the withdrawn
|
||||
// fieldset; the tombstone makes their next HIMPORT command discard it
|
||||
// instead of leaving a fieldset the client can no longer address.
|
||||
func (r *himportRegistry) discardVersion(name string, version uint64) {
|
||||
r.mu.Lock()
|
||||
if fs, ok := r.fieldsets[name]; ok && fs.version == version {
|
||||
delete(r.fieldsets, name)
|
||||
if r.tombstones == nil {
|
||||
r.tombstones = make(map[string]struct{})
|
||||
}
|
||||
r.tombstones[name] = struct{}{}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// refreshVersion bumps a registered fieldset to a new version, keeping its
|
||||
// fields — but only while the entry is still at the given version, so a
|
||||
// concurrent re-registration is not disturbed. Every connection's prepared
|
||||
// flag becomes stale, forcing a re-prepare before the fieldset's next use on
|
||||
// each of them. Used when a "no such fieldset" reply signals session loss
|
||||
// that may have hit more connections than the one that reported it (failover,
|
||||
// cross-region switch, reset storms).
|
||||
func (r *himportRegistry) refreshVersion(name string, version uint64) {
|
||||
r.mu.Lock()
|
||||
if fs, ok := r.fieldsets[name]; ok && fs.version == version {
|
||||
r.nextVersion++
|
||||
fs.version = r.nextVersion
|
||||
r.fieldsets[name] = fs
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// idle reports whether the registry implies no injection work at all: no
|
||||
// fieldsets to replay, no tombstones to discard, and no discard-all epoch a
|
||||
// session could be behind.
|
||||
func (r *himportRegistry) idle() bool {
|
||||
if r == nil {
|
||||
return true
|
||||
}
|
||||
r.mu.RLock()
|
||||
idle := len(r.fieldsets) == 0 && len(r.tombstones) == 0 && r.discardAllEpoch == 0
|
||||
r.mu.RUnlock()
|
||||
return idle
|
||||
}
|
||||
|
||||
// cleanupSnapshot returns the current epoch and the tombstoned names.
|
||||
func (r *himportRegistry) cleanupSnapshot() (epoch uint64, tombstones []string) {
|
||||
r.mu.RLock()
|
||||
epoch = r.discardAllEpoch
|
||||
if len(r.tombstones) > 0 {
|
||||
tombstones = make([]string, 0, len(r.tombstones))
|
||||
for name := range r.tombstones {
|
||||
tombstones = append(tombstones, name)
|
||||
}
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
return epoch, tombstones
|
||||
}
|
||||
|
||||
// himportNoSuchFieldset reports whether err is the server's "no such
|
||||
// fieldset" reply, i.e. an HIMPORT SET executed on a connection whose session
|
||||
// does not hold the referenced fieldset.
|
||||
func himportNoSuchFieldset(err error) bool {
|
||||
return isRedisError(err) && strings.Contains(err.Error(), "no such fieldset")
|
||||
}
|
||||
|
||||
// himportInjectedCmds returns the HIMPORT commands to write to cn ahead of a
|
||||
// batch, in order:
|
||||
//
|
||||
// 1. HIMPORT DISCARDALL when cn's session was prepared under an older
|
||||
// discard-all epoch;
|
||||
// 2. HIMPORT DISCARD for each discarded fieldset the session still holds;
|
||||
// 3. HIMPORT PREPARE for each registered fieldset referenced by an HIMPORT
|
||||
// SET in the batch that the session lacks at the current version.
|
||||
//
|
||||
// A fieldset covered by a user-issued PREPARE earlier in the batch needs no
|
||||
// injection — the server session holds it by the time the SET runs. Returns
|
||||
// nil when the batch contains no HIMPORT commands: sessions holding only
|
||||
// discarded fieldsets are cleaned up on their next HIMPORT use, not on
|
||||
// unrelated traffic.
|
||||
func (c *baseClient) himportInjectedCmds(ctx context.Context, cn *pool.Conn, cmds []Cmder) []Cmder {
|
||||
if c.himport.idle() {
|
||||
return nil
|
||||
}
|
||||
hasHImport := false
|
||||
for _, cmd := range cmds {
|
||||
if _, ok := cmd.(himportCmder); ok {
|
||||
hasHImport = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasHImport {
|
||||
return nil
|
||||
}
|
||||
|
||||
var injected []Cmder
|
||||
|
||||
// Discards first: a session behind the discard-all epoch is wiped
|
||||
// entirely; otherwise individual tombstoned fieldsets it still holds are
|
||||
// discarded.
|
||||
epoch, tombstones := c.himport.cleanupSnapshot()
|
||||
sessionWiped := false
|
||||
if cn.HasPreparedFieldsets() && cn.FieldsetEpoch() != epoch {
|
||||
da := NewHImportDiscardAllCmd(ctx)
|
||||
da.registryEpoch = epoch
|
||||
injected = append(injected, da)
|
||||
sessionWiped = true
|
||||
} else {
|
||||
for _, name := range tombstones {
|
||||
if cn.FieldsetPreparedVersion(name) != 0 {
|
||||
injected = append(injected, NewHImportDiscardCmd(ctx, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prepares for registered fieldsets the batch's SETs reference.
|
||||
var covered map[string]struct{}
|
||||
cover := func(name string) {
|
||||
if covered == nil {
|
||||
covered = make(map[string]struct{})
|
||||
}
|
||||
covered[name] = struct{}{}
|
||||
}
|
||||
for _, cmd := range cmds {
|
||||
switch hc := cmd.(type) {
|
||||
case *HImportPrepareCmd:
|
||||
cover(hc.fieldsetName)
|
||||
case *HImportSetCmd:
|
||||
if _, ok := covered[hc.fieldsetName]; ok {
|
||||
continue
|
||||
}
|
||||
fs, ok := c.himport.lookup(hc.fieldsetName)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !sessionWiped && cn.FieldsetPreparedVersion(hc.fieldsetName) == fs.version {
|
||||
continue
|
||||
}
|
||||
// The session holds an older version. Discard it before the
|
||||
// re-prepare: the SET behind it is already on the wire, and if
|
||||
// the re-prepare fails the SET must answer "no such fieldset"
|
||||
// rather than silently writing the old version's field names.
|
||||
if !sessionWiped && cn.FieldsetPreparedVersion(hc.fieldsetName) != 0 {
|
||||
injected = append(injected, NewHImportDiscardCmd(ctx, hc.fieldsetName))
|
||||
}
|
||||
prep := NewHImportPrepareCmd(ctx, hc.fieldsetName, fs.fields...)
|
||||
prep.registryVersion = fs.version
|
||||
prep.registryEpoch = epoch
|
||||
injected = append(injected, prep)
|
||||
cover(hc.fieldsetName)
|
||||
}
|
||||
}
|
||||
return injected
|
||||
}
|
||||
|
||||
// himportReadInjectedReplies consumes the replies of injected HIMPORT
|
||||
// commands. Server errors are recorded on the command and the connection is
|
||||
// left readable; transport errors are returned. Successful commands apply
|
||||
// their prepared-flag bookkeeping on cn.
|
||||
func (c *baseClient) himportReadInjectedReplies(ctx context.Context, cn *pool.Conn, rd *proto.Reader, injected []Cmder) error {
|
||||
for _, cmd := range injected {
|
||||
if err := c.processPendingPushNotificationWithReader(ctx, cn, rd); err != nil {
|
||||
internal.Logger.Printf(ctx, "push: error processing pending notifications before reading reply: %v", err)
|
||||
}
|
||||
err := cmd.readReply(rd)
|
||||
cmd.SetErr(err)
|
||||
if err != nil {
|
||||
if !isRedisError(err) {
|
||||
return err
|
||||
}
|
||||
// A failed injected PREPARE becomes the root cause of the
|
||||
// dependent SETs' errors downstream; a failed injected discard
|
||||
// only delays cleanup until the next HIMPORT command.
|
||||
internal.Logger.Printf(ctx, "himport: injected %s failed: %v", cmd.Name(), err)
|
||||
continue
|
||||
}
|
||||
switch hc := cmd.(type) {
|
||||
case *HImportPrepareCmd:
|
||||
cn.MarkFieldsetPrepared(hc.fieldsetName, hc.registryVersion, hc.registryEpoch)
|
||||
case *HImportDiscardCmd:
|
||||
cn.UnmarkFieldsetPrepared(hc.fieldsetName)
|
||||
case *HImportDiscardAllCmd:
|
||||
cn.ClearPreparedFieldsets(hc.registryEpoch)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// himportAfterCmd applies registry and prepared-flag updates after a
|
||||
// user-issued HIMPORT command completed successfully on cn.
|
||||
func (c *baseClient) himportAfterCmd(cn *pool.Conn, hc himportCmder) {
|
||||
if c.himport == nil {
|
||||
return
|
||||
}
|
||||
switch cmd := hc.(type) {
|
||||
case *HImportPrepareCmd:
|
||||
version, epoch := cmd.registryVersion, cmd.registryEpoch
|
||||
if version == 0 {
|
||||
version, epoch = c.himport.register(cmd.fieldsetName, cmd.fields)
|
||||
}
|
||||
// A pre-assigned version marks a fan-out copy: the fieldset was
|
||||
// registered once at the cluster/ring level; only mark the
|
||||
// executing connection.
|
||||
cn.MarkFieldsetPrepared(cmd.fieldsetName, version, epoch)
|
||||
case *HImportDiscardCmd:
|
||||
registered := c.himport.discard(cmd.fieldsetName)
|
||||
cn.UnmarkFieldsetPrepared(cmd.fieldsetName)
|
||||
// The managed API reports the registry lifecycle: 1 when the
|
||||
// fieldset was registered on this client and is now removed. The
|
||||
// executing connection's session count stands only for fieldsets
|
||||
// the registry never knew (raw usage).
|
||||
if registered {
|
||||
cmd.SetVal(1)
|
||||
}
|
||||
case *HImportDiscardAllCmd:
|
||||
// A pre-assigned epoch marks a fan-out copy: the registry was
|
||||
// already wiped at the cluster/ring level; only move the executing
|
||||
// connection to that epoch.
|
||||
if cmd.registryEpoch != 0 {
|
||||
cn.ClearPreparedFieldsets(cmd.registryEpoch)
|
||||
return
|
||||
}
|
||||
epoch, removed := c.himport.discardAll()
|
||||
cn.ClearPreparedFieldsets(epoch)
|
||||
// Same registry semantics: report how many registered fieldsets
|
||||
// were removed, not how many the executing session happened to
|
||||
// hold.
|
||||
if removed > 0 {
|
||||
cmd.SetVal(int64(removed))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// himportAfterBatch runs after all replies of a batch were read: it surfaces
|
||||
// an injected PREPARE failure as the root cause on the HIMPORT SET commands
|
||||
// that depended on it (their own reply is the secondary "no such fieldset"
|
||||
// error), invalidates stale prepared flags for SETs that found their
|
||||
// registered fieldset missing server-side, and applies registry updates for
|
||||
// user-issued HIMPORT commands that succeeded in the batch.
|
||||
// rawErr throughout: this runs on the execution path, before an async
|
||||
// autopipeline batch completes (its ready channel closes only after the
|
||||
// pipeline hook chain returns) — Err() on a user command would await and
|
||||
// self-deadlock the dispatcher.
|
||||
func (c *baseClient) himportAfterBatch(cn *pool.Conn, injected []Cmder, cmds []Cmder) {
|
||||
var failed map[string]error
|
||||
var refreshed map[string]struct{}
|
||||
for _, cmd := range injected {
|
||||
if prep, ok := cmd.(*HImportPrepareCmd); ok {
|
||||
if err := prep.Err(); err != nil {
|
||||
if failed == nil {
|
||||
failed = make(map[string]error)
|
||||
}
|
||||
failed[prep.fieldsetName] = err
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, cmd := range cmds {
|
||||
hc, ok := cmd.(himportCmder)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if set, ok := hc.(*HImportSetCmd); ok {
|
||||
if rootCause, ok := failed[set.fieldsetName]; ok && himportNoSuchFieldset(set.rawErr()) {
|
||||
set.SetErr(rootCause)
|
||||
continue
|
||||
}
|
||||
// The session lost a fieldset the flags claim is prepared (e.g.
|
||||
// RESET) — and the same event may have wiped other sessions
|
||||
// whose flags also still look current. Bump the fieldset
|
||||
// version once so the SET's re-issue, the cluster re-queue on
|
||||
// whichever connection it lands, or the caller's transaction
|
||||
// retry replays the PREPARE.
|
||||
if himportNoSuchFieldset(set.rawErr()) {
|
||||
if _, done := refreshed[set.fieldsetName]; !done {
|
||||
if refreshed == nil {
|
||||
refreshed = make(map[string]struct{})
|
||||
}
|
||||
refreshed[set.fieldsetName] = struct{}{}
|
||||
if fs, registered := c.himport.lookup(set.fieldsetName); registered {
|
||||
c.himport.refreshVersion(set.fieldsetName, fs.version)
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if hc.rawErr() == nil {
|
||||
c.himportAfterCmd(cn, hc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// himportRetryFailedSets re-issues, once, the HIMPORT SET commands of a
|
||||
// pipeline batch that failed with "no such fieldset" while their fieldset is
|
||||
// registered — the error must not surface for managed fieldsets (NF.4). Only
|
||||
// the SETs are re-sent: HIMPORT SET is a full replace, so re-execution is
|
||||
// idempotent, and no other command of the batch runs again. Their prepared
|
||||
// flags were invalidated by himportAfterBatch, so himportInjectedCmds
|
||||
// regenerates the PREPAREs for this connection. Transport errors are
|
||||
// returned; server errors stay recorded on the commands.
|
||||
// (The retry does not carry an ASKING prefix. A redirected [ASKING, SET]
|
||||
// pair whose injected PREPARE failed is excluded by the root-cause swap in
|
||||
// himportAfterBatch; one that lost its session without an injection can be
|
||||
// re-issued here, and the bare SET then draws a fresh MOVED/ASK that the
|
||||
// outer cluster redirect handling resolves.)
|
||||
func (c *baseClient) himportRetryFailedSets(ctx context.Context, cn *pool.Conn, cmds []Cmder) error {
|
||||
if c.himport.idle() {
|
||||
return nil
|
||||
}
|
||||
var retry []Cmder
|
||||
for _, cmd := range cmds {
|
||||
// rawErr: execution path, same self-deadlock rule as himportAfterBatch.
|
||||
if set, ok := cmd.(*HImportSetCmd); ok && himportNoSuchFieldset(set.rawErr()) {
|
||||
if _, registered := c.himport.lookup(set.fieldsetName); registered {
|
||||
retry = append(retry, set)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(retry) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
injected := c.himportInjectedCmds(ctx, cn, retry)
|
||||
if err := cn.WithWriter(c.context(ctx), c.opt.WriteTimeout, func(wr *proto.Writer) error {
|
||||
for _, ic := range injected {
|
||||
if err := writeCmd(wr, ic); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writeCmds(wr, retry)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return cn.WithReader(c.context(ctx), c.opt.ReadTimeout, func(rd *proto.Reader) error {
|
||||
if err := c.himportReadInjectedReplies(ctx, cn, rd, injected); err != nil {
|
||||
return err
|
||||
}
|
||||
err := c.pipelineReadCmds(ctx, cn, rd, retry)
|
||||
if err != nil && !isRedisError(err) {
|
||||
return err
|
||||
}
|
||||
// Server errors (including a repeated failure) stay on the
|
||||
// individual commands; the batch as a whole is done.
|
||||
c.himportAfterBatch(cn, injected, retry)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// himportShouldRetrySet reports whether a retry of cmd may succeed after it
|
||||
// failed with "no such fieldset": true when the fieldset is registered
|
||||
// client-side — the executing connection lost its server session state (for
|
||||
// example a RESET, or a concurrent discard). The connection's prepared flag
|
||||
// was already invalidated inside _process, while that goroutine still owned
|
||||
// the connection, so the retry re-prepares lazily wherever it lands.
|
||||
func (c *baseClient) himportShouldRetrySet(cmd Cmder, err error) bool {
|
||||
set, ok := cmd.(*HImportSetCmd)
|
||||
if !ok || !himportNoSuchFieldset(err) {
|
||||
return false
|
||||
}
|
||||
_, registered := c.himport.lookup(set.fieldsetName)
|
||||
return registered
|
||||
}
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
package redis
|
||||
|
||||
import "context"
|
||||
|
||||
// Cluster and ring support for the HIMPORT command family.
|
||||
//
|
||||
// Correctness comes from the shared registry: every node/shard client holds
|
||||
// the same himportRegistry (wired at client construction), so any connection
|
||||
// executing an HIMPORT SET lazily replays the PREPARE, MOVED/ASK redirects
|
||||
// re-prepare on the target node, and discards propagate through tombstones
|
||||
// and the discard-all epoch. Replicas share the registry too — roles change
|
||||
// with the topology, and a promoted replica's connections simply carry no
|
||||
// prepared flags, so their first SET self-prepares.
|
||||
//
|
||||
// On top of that, user-issued PREPARE/DISCARD/DISCARDALL fan out eagerly to
|
||||
// all masters (R.4): one connection per master is prepared/cleaned up front,
|
||||
// server-side validation surfaces immediately, and leftover session state is
|
||||
// bounded. The fan-out is best-effort — any connection it does not reach is
|
||||
// covered by the lazy replay.
|
||||
|
||||
// The fan-out helpers execute the per-node copies through each node client's
|
||||
// Process, so node-level hooks observe them; the cluster/ring-level
|
||||
// ProcessHook chain sees only the user's command object, not the fan-out.
|
||||
//
|
||||
// Known limitation: an HImportPrepare pipelined together with HImportSets of
|
||||
// the same new fieldset in one ClusterClient Exec is not ordered across
|
||||
// nodes — per-node sub-batches run concurrently, and the registration
|
||||
// happens when the PREPARE's node completes, so SETs routed to other nodes
|
||||
// can race it and fail with "no such fieldset". Register the fieldset with
|
||||
// the client-level HImportPrepare before pipelining (the HLD's back-to-back
|
||||
// PREPARE+SET pattern is a single-connection guarantee).
|
||||
|
||||
// himportForEach runs fn on a set of clients (all cluster masters, or all
|
||||
// ring shards).
|
||||
type himportForEach func(ctx context.Context, fn func(ctx context.Context, client *Client) error) error
|
||||
|
||||
// himportRequeueFailedSets re-queues HIMPORT SETs of registered fieldsets
|
||||
// that failed with "no such fieldset" — their stale prepared flags were just
|
||||
// invalidated by himportAfterBatch, so the next pipeline attempt re-prepares
|
||||
// lazily and re-executes only those SETs (a full replace, so idempotent).
|
||||
// Bounded by the cluster pipeline's attempt budget.
|
||||
func (c *ClusterClient) himportRequeueFailedSets(ctx context.Context, cmds []Cmder, failedCmds *cmdsMap) {
|
||||
for _, cmd := range cmds {
|
||||
// rawErr: runs on the per-node execution goroutine, same
|
||||
// self-deadlock rule as himportAfterBatch.
|
||||
if set, ok := cmd.(*HImportSetCmd); ok && himportNoSuchFieldset(set.rawErr()) {
|
||||
if _, registered := c.himport.lookup(set.fieldsetName); registered {
|
||||
_ = c.mapCmdsByNode(ctx, failedCmds, []Cmder{set})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// himportFanOutPrepare registers the fieldset once in the shared registry
|
||||
// and executes a pre-versioned PREPARE copy on every client; each copy marks
|
||||
// its executing connection without registering again. A deterministic server
|
||||
// rejection (e.g. duplicate field name) withdraws the registration; a
|
||||
// transport failure keeps it, and lazy replay covers the connections the
|
||||
// fan-out missed (all-succeeded semantics: the first error is reported).
|
||||
func himportFanOutPrepare(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportPrepareCmd) {
|
||||
version, epoch := registry.register(cmd.fieldsetName, cmd.fields)
|
||||
err := forEach(ctx, func(ctx context.Context, client *Client) error {
|
||||
fanCmd := NewHImportPrepareCmd(ctx, cmd.fieldsetName, cmd.fields...)
|
||||
fanCmd.registryVersion = version
|
||||
fanCmd.registryEpoch = epoch
|
||||
return client.Process(ctx, fanCmd)
|
||||
})
|
||||
if err != nil {
|
||||
if isRedisError(err) {
|
||||
// Withdraw the registration; the tombstone cleans the sessions
|
||||
// on which the fan-out succeeded before the rejection.
|
||||
registry.discardVersion(cmd.fieldsetName, version)
|
||||
}
|
||||
cmd.SetErr(err)
|
||||
return
|
||||
}
|
||||
cmd.SetVal("OK")
|
||||
}
|
||||
|
||||
// himportFanOutDiscard removes the fieldset from the shared registry
|
||||
// (leaving the tombstone that lazily cleans the connections the fan-out does
|
||||
// not reach) and discards it on one connection of every client.
|
||||
func himportFanOutDiscard(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportDiscardCmd) {
|
||||
registered := registry.discard(cmd.fieldsetName)
|
||||
err := forEach(ctx, func(ctx context.Context, client *Client) error {
|
||||
return client.Process(ctx, NewHImportDiscardCmd(ctx, cmd.fieldsetName))
|
||||
})
|
||||
if err != nil {
|
||||
cmd.SetErr(err)
|
||||
return
|
||||
}
|
||||
if registered {
|
||||
cmd.SetVal(1)
|
||||
} else {
|
||||
cmd.SetVal(0)
|
||||
}
|
||||
}
|
||||
|
||||
// himportFanOutDiscardAll wipes the shared registry once and executes a
|
||||
// pre-epoch DISCARDALL copy on every client; each copy moves its executing
|
||||
// connection to the new epoch without bumping the registry again.
|
||||
func himportFanOutDiscardAll(ctx context.Context, registry *himportRegistry, forEach himportForEach, cmd *HImportDiscardAllCmd) {
|
||||
epoch, removed := registry.discardAll()
|
||||
err := forEach(ctx, func(ctx context.Context, client *Client) error {
|
||||
fanCmd := NewHImportDiscardAllCmd(ctx)
|
||||
fanCmd.registryEpoch = epoch
|
||||
return client.Process(ctx, fanCmd)
|
||||
})
|
||||
if err != nil {
|
||||
cmd.SetErr(err)
|
||||
return
|
||||
}
|
||||
cmd.SetVal(int64(removed))
|
||||
}
|
||||
|
||||
// HImportPrepare registers the fieldset in the cluster-wide registry and
|
||||
// eagerly prepares one connection on every master; all other connections —
|
||||
// including those of replicas promoted later and masters added by
|
||||
// resharding — are prepared lazily before their first HImportSet. See
|
||||
// HashCmdable.HImportPrepare (cmdable) for the fieldset semantics.
|
||||
//
|
||||
// The fan-out is best-effort and reports the first error: on a server
|
||||
// rejection (e.g. duplicate field name) the registration is withdrawn and
|
||||
// any sessions the fan-out already prepared are cleaned lazily; on a
|
||||
// transport failure the registration is kept and lazy replay covers the
|
||||
// connections the fan-out missed.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c *ClusterClient) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd {
|
||||
cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...)
|
||||
himportFanOutPrepare(ctx, c.himport, c.ForEachMaster, cmd)
|
||||
return &cmd.StatusCmd
|
||||
}
|
||||
|
||||
// HImportDiscard removes the fieldset from the cluster-wide registry and
|
||||
// discards it on every master; connections the fan-out does not reach
|
||||
// replay the discard before their next HIMPORT command. It returns 1 if the
|
||||
// fieldset was registered on this client and is now removed.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c *ClusterClient) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd {
|
||||
cmd := NewHImportDiscardCmd(ctx, fieldsetName)
|
||||
himportFanOutDiscard(ctx, c.himport, c.ForEachMaster, cmd)
|
||||
return &cmd.IntCmd
|
||||
}
|
||||
|
||||
// HImportDiscardAll removes all fieldsets from the cluster-wide registry and
|
||||
// wipes them on every master; connections the fan-out does not reach replay
|
||||
// the wipe before their next HIMPORT command. It returns the number of
|
||||
// fieldsets removed from the registry.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c *ClusterClient) HImportDiscardAll(ctx context.Context) *IntCmd {
|
||||
cmd := NewHImportDiscardAllCmd(ctx)
|
||||
himportFanOutDiscardAll(ctx, c.himport, c.ForEachMaster, cmd)
|
||||
return &cmd.IntCmd
|
||||
}
|
||||
|
||||
// HImportPrepare registers the fieldset in the ring-wide registry and
|
||||
// eagerly prepares one connection on every shard; all other connections are
|
||||
// prepared lazily before their first HImportSet. The fan-out is best-effort
|
||||
// with the same failure semantics as ClusterClient.HImportPrepare. See
|
||||
// HashCmdable.HImportPrepare (cmdable) for the fieldset semantics.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c *Ring) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd {
|
||||
cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...)
|
||||
himportFanOutPrepare(ctx, c.opt.himport, c.ForEachShard, cmd)
|
||||
return &cmd.StatusCmd
|
||||
}
|
||||
|
||||
// HImportDiscard removes the fieldset from the ring-wide registry and
|
||||
// discards it on every shard; connections the fan-out does not reach replay
|
||||
// the discard before their next HIMPORT command. It returns 1 if the
|
||||
// fieldset was registered on this client and is now removed.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c *Ring) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd {
|
||||
cmd := NewHImportDiscardCmd(ctx, fieldsetName)
|
||||
himportFanOutDiscard(ctx, c.opt.himport, c.ForEachShard, cmd)
|
||||
return &cmd.IntCmd
|
||||
}
|
||||
|
||||
// HImportDiscardAll removes all fieldsets from the ring-wide registry and
|
||||
// wipes them on every shard; connections the fan-out does not reach replay
|
||||
// the wipe before their next HIMPORT command. It returns the number of
|
||||
// fieldsets removed from the registry.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c *Ring) HImportDiscardAll(ctx context.Context) *IntCmd {
|
||||
cmd := NewHImportDiscardAllCmd(ctx)
|
||||
himportFanOutDiscardAll(ctx, c.opt.himport, c.ForEachShard, cmd)
|
||||
return &cmd.IntCmd
|
||||
}
|
||||
+253
@@ -0,0 +1,253 @@
|
||||
package redis
|
||||
|
||||
import "context"
|
||||
|
||||
// The HIMPORT command family (Redis 8.10+, "hinted hash templates") provides
|
||||
// fast ingestion of many hashes sharing the same field names. HIMPORT PREPARE
|
||||
// registers the field names once under a fieldset name, then HIMPORT SET
|
||||
// creates hashes by sending only the values.
|
||||
//
|
||||
// The server scopes a fieldset to the physical connection that prepared it.
|
||||
// Because go-redis pools connections, the client additionally keeps a
|
||||
// client-side registry of fieldsets registered through HImportPrepare and
|
||||
// lazily replays the PREPARE (at most once per connection session) on any
|
||||
// pooled connection about to execute an HImportSet that references it. See
|
||||
// himport.go.
|
||||
//
|
||||
// The whole HIMPORT surface — the typed methods, the HImport*Cmd types and
|
||||
// their constructors — is experimental and may be subject to change.
|
||||
|
||||
// himportCmder marks HIMPORT commands that participate in client-side
|
||||
// fieldset tracking. Process paths do a single interface assertion on the
|
||||
// hot path and inspect the concrete type only for HIMPORT commands.
|
||||
type himportCmder interface {
|
||||
Cmder
|
||||
himportCmd()
|
||||
}
|
||||
|
||||
var (
|
||||
_ himportCmder = (*HImportPrepareCmd)(nil)
|
||||
_ himportCmder = (*HImportSetCmd)(nil)
|
||||
_ himportCmder = (*HImportDiscardCmd)(nil)
|
||||
_ himportCmder = (*HImportDiscardAllCmd)(nil)
|
||||
)
|
||||
|
||||
// HImportPrepareCmd represents an HIMPORT PREPARE command.
|
||||
type HImportPrepareCmd struct {
|
||||
StatusCmd
|
||||
|
||||
fieldsetName string
|
||||
fields []string
|
||||
|
||||
// registryVersion and registryEpoch are set only on commands injected by
|
||||
// the client to replay a registered fieldset onto a connection; on
|
||||
// success the connection is marked as prepared at this version under
|
||||
// this discard-all epoch.
|
||||
registryVersion uint64
|
||||
registryEpoch uint64
|
||||
}
|
||||
|
||||
func (cmd *HImportPrepareCmd) himportCmd() {}
|
||||
|
||||
// NewHImportPrepareCmd returns an HIMPORT PREPARE command.
|
||||
func NewHImportPrepareCmd(ctx context.Context, fieldsetName string, fields ...string) *HImportPrepareCmd {
|
||||
args := make([]interface{}, 3+len(fields))
|
||||
args[0] = "himport"
|
||||
args[1] = "prepare"
|
||||
args[2] = fieldsetName
|
||||
for i, field := range fields {
|
||||
args[3+i] = field
|
||||
}
|
||||
return &HImportPrepareCmd{
|
||||
StatusCmd: StatusCmd{
|
||||
baseCmd: baseCmd{
|
||||
ctx: ctx,
|
||||
args: args,
|
||||
cmdType: CmdTypeStatus,
|
||||
},
|
||||
},
|
||||
fieldsetName: fieldsetName,
|
||||
fields: append([]string(nil), fields...),
|
||||
}
|
||||
}
|
||||
|
||||
// HImportSetCmd represents an HIMPORT SET command.
|
||||
type HImportSetCmd struct {
|
||||
StatusCmd
|
||||
|
||||
fieldsetName string
|
||||
}
|
||||
|
||||
func (cmd *HImportSetCmd) himportCmd() {}
|
||||
|
||||
// NewHImportSetCmd returns an HIMPORT SET command.
|
||||
func NewHImportSetCmd(ctx context.Context, key, fieldsetName string, values ...interface{}) *HImportSetCmd {
|
||||
args := make([]interface{}, 4+len(values))
|
||||
args[0] = "himport"
|
||||
args[1] = "set"
|
||||
args[2] = key
|
||||
args[3] = fieldsetName
|
||||
copy(args[4:], values)
|
||||
cmd := &HImportSetCmd{
|
||||
StatusCmd: StatusCmd{
|
||||
baseCmd: baseCmd{
|
||||
ctx: ctx,
|
||||
args: args,
|
||||
cmdType: CmdTypeStatus,
|
||||
},
|
||||
},
|
||||
fieldsetName: fieldsetName,
|
||||
}
|
||||
cmd.SetFirstKeyPos(2)
|
||||
return cmd
|
||||
}
|
||||
|
||||
// HImportDiscardCmd represents an HIMPORT DISCARD command.
|
||||
type HImportDiscardCmd struct {
|
||||
IntCmd
|
||||
|
||||
fieldsetName string
|
||||
}
|
||||
|
||||
func (cmd *HImportDiscardCmd) himportCmd() {}
|
||||
|
||||
// NewHImportDiscardCmd returns an HIMPORT DISCARD command.
|
||||
func NewHImportDiscardCmd(ctx context.Context, fieldsetName string) *HImportDiscardCmd {
|
||||
return &HImportDiscardCmd{
|
||||
IntCmd: IntCmd{
|
||||
baseCmd: baseCmd{
|
||||
ctx: ctx,
|
||||
args: []interface{}{"himport", "discard", fieldsetName},
|
||||
cmdType: CmdTypeInt,
|
||||
},
|
||||
},
|
||||
fieldsetName: fieldsetName,
|
||||
}
|
||||
}
|
||||
|
||||
// HImportDiscardAllCmd represents an HIMPORT DISCARDALL command.
|
||||
type HImportDiscardAllCmd struct {
|
||||
IntCmd
|
||||
|
||||
// registryEpoch is set only on commands injected by the client to wipe a
|
||||
// session that predates the registry's discard-all epoch; on success the
|
||||
// connection adopts this epoch.
|
||||
registryEpoch uint64
|
||||
}
|
||||
|
||||
func (cmd *HImportDiscardAllCmd) himportCmd() {}
|
||||
|
||||
// NewHImportDiscardAllCmd returns an HIMPORT DISCARDALL command.
|
||||
func NewHImportDiscardAllCmd(ctx context.Context) *HImportDiscardAllCmd {
|
||||
return &HImportDiscardAllCmd{
|
||||
IntCmd: IntCmd{
|
||||
baseCmd: baseCmd{
|
||||
ctx: ctx,
|
||||
args: []interface{}{"himport", "discardall"},
|
||||
cmdType: CmdTypeInt,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// HImportPrepare registers an ordered list of hash field names under
|
||||
// fieldsetName for use by subsequent HImportSet calls:
|
||||
//
|
||||
// HIMPORT PREPARE fieldset_name field [field ...]
|
||||
//
|
||||
// The server keeps the fieldset in the session of the connection that
|
||||
// executed the command. On pooled clients (Client, Conn, Pipeline, Tx) the
|
||||
// fieldset is also remembered client-side and the PREPARE is replayed
|
||||
// lazily — at most once per connection session — on any pooled connection
|
||||
// about to execute an HImportSet referencing it, so HImportSet works
|
||||
// transparently across the pool. Preparing an existing fieldset name again
|
||||
// silently replaces it.
|
||||
//
|
||||
// ClusterClient and Ring override this method (see himport_cluster.go): the
|
||||
// fieldset registers in a registry shared by every node/shard client and the
|
||||
// PREPARE additionally fans out eagerly to all masters/shards.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c cmdable) HImportPrepare(ctx context.Context, fieldsetName string, fields ...string) *StatusCmd {
|
||||
cmd := NewHImportPrepareCmd(ctx, fieldsetName, fields...)
|
||||
_ = c(ctx, cmd)
|
||||
return &cmd.StatusCmd
|
||||
}
|
||||
|
||||
// HImportSet creates or fully replaces the hash at key using the field list
|
||||
// registered under fieldsetName, pairing values positionally with the
|
||||
// prepared fields:
|
||||
//
|
||||
// HIMPORT SET key fieldset_name value [value ...]
|
||||
//
|
||||
// The number of values must equal the fieldset's field count. The resulting
|
||||
// key is a regular hash readable and writable by all hash commands. If the
|
||||
// fieldset was registered through HImportPrepare on this client, it is
|
||||
// prepared automatically on whichever pooled connection executes the command;
|
||||
// otherwise the fieldset must have been prepared on the executing connection
|
||||
// or the server replies "ERR no such fieldset".
|
||||
//
|
||||
// "no such fieldset" never surfaces for a registered fieldset: a
|
||||
// single-command HImportSet whose connection lost its session state (e.g.
|
||||
// RESET) is transparently re-prepared and retried once — the failure also
|
||||
// stales every other connection's prepared flag, so the retry re-prepares
|
||||
// wherever it lands, and this recovery attempt is granted even when retries
|
||||
// are disabled (MaxRetries -1). In pipelines the failed HImportSets — and
|
||||
// only those — are re-prepared and re-issued once on the same connection
|
||||
// (HIMPORT SET is a full replace, so the re-execution is idempotent and no
|
||||
// other command of the batch runs again). Inside transactions the error does
|
||||
// surface after EXEC — an executed transaction cannot be partially re-run —
|
||||
// but the prepared flags are invalidated, so retrying the transaction
|
||||
// succeeds.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c cmdable) HImportSet(ctx context.Context, key, fieldsetName string, values ...interface{}) *StatusCmd {
|
||||
cmd := NewHImportSetCmd(ctx, key, fieldsetName, values...)
|
||||
_ = c(ctx, cmd)
|
||||
return &cmd.StatusCmd
|
||||
}
|
||||
|
||||
// HImportDiscard removes fieldsetName from the executing connection's session
|
||||
// and from the client-side registry, stopping further automatic replay:
|
||||
//
|
||||
// HIMPORT DISCARD fieldset_name
|
||||
//
|
||||
// It returns 1 if the fieldset was registered on this client and is now
|
||||
// removed, 0 otherwise (for names never registered through the managed API,
|
||||
// the executing connection's session reply passes through unchanged). Pooled
|
||||
// connections whose sessions still hold the fieldset replay the DISCARD
|
||||
// before their next HIMPORT command, so a subsequent HImportSet fails with
|
||||
// "no such fieldset" on every connection, exactly as on a single connection.
|
||||
// Hashes already created through the fieldset are not affected.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c cmdable) HImportDiscard(ctx context.Context, fieldsetName string) *IntCmd {
|
||||
cmd := NewHImportDiscardCmd(ctx, fieldsetName)
|
||||
_ = c(ctx, cmd)
|
||||
return &cmd.IntCmd
|
||||
}
|
||||
|
||||
// HImportDiscardAll removes all fieldsets from the executing connection's
|
||||
// session and clears the client-side registry:
|
||||
//
|
||||
// HIMPORT DISCARDALL
|
||||
//
|
||||
// It returns the number of fieldsets removed from the client-side registry
|
||||
// (when none were registered, the executing connection's session count
|
||||
// passes through). Other pooled connections whose sessions were prepared
|
||||
// earlier replay HIMPORT DISCARDALL before their next HIMPORT command.
|
||||
//
|
||||
// Requires Redis 8.10 or newer.
|
||||
//
|
||||
// note: the API is experimental and may be subject to change.
|
||||
func (c cmdable) HImportDiscardAll(ctx context.Context) *IntCmd {
|
||||
cmd := NewHImportDiscardAllCmd(ctx)
|
||||
_ = c(ctx, cmd)
|
||||
return &cmd.IntCmd
|
||||
}
|
||||
+4
-4
@@ -27,7 +27,7 @@ import (
|
||||
// and is re-armed on failure.
|
||||
type Once struct {
|
||||
m sync.Mutex
|
||||
done uint32
|
||||
done atomic.Uint32
|
||||
}
|
||||
|
||||
// Do calls the function f if and only if Do has not been invoked
|
||||
@@ -46,17 +46,17 @@ type Once struct {
|
||||
//
|
||||
// err := config.once.Do(func() error { return config.init(filename) })
|
||||
func (o *Once) Do(f func() error) error {
|
||||
if atomic.LoadUint32(&o.done) == 1 {
|
||||
if o.done.Load() == 1 {
|
||||
return nil
|
||||
}
|
||||
// Slow-path.
|
||||
o.m.Lock()
|
||||
defer o.m.Unlock()
|
||||
var err error
|
||||
if o.done == 0 {
|
||||
if o.done.Load() == 0 {
|
||||
err = f()
|
||||
if err == nil {
|
||||
atomic.StoreUint32(&o.done, 1)
|
||||
o.done.Store(1)
|
||||
}
|
||||
}
|
||||
return err
|
||||
|
||||
+31
-8
@@ -155,6 +155,15 @@ func getRecorder() Recorder {
|
||||
return r
|
||||
}
|
||||
|
||||
// Enabled reports whether a real recorder is installed. Callers use it to
|
||||
// skip metric work whose INPUTS are expensive to obtain — e.g. reading a
|
||||
// command's result, which on the async autopipeline face blocks until the
|
||||
// command executes.
|
||||
func Enabled() bool {
|
||||
_, noop := getRecorder().(noopRecorder)
|
||||
return !noop
|
||||
}
|
||||
|
||||
// SetGlobalRecorder sets the global recorder (called by Init() in extra/redisotel-native)
|
||||
func SetGlobalRecorder(r Recorder) {
|
||||
recorderMu.Lock()
|
||||
@@ -266,10 +275,15 @@ func (noopRecorder) RecordStreamLag(context.Context, time.Duration, *pool.Conn,
|
||||
func (noopRecorder) RecordConnectionCount(context.Context, int, *pool.Conn, string, bool) {}
|
||||
func (noopRecorder) RecordPendingRequests(context.Context, int, *pool.Conn, string) {}
|
||||
|
||||
// RegisterPools registers connection pools with the global recorder.
|
||||
func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, addr string) {
|
||||
// Check if the global recorder implements PoolRegistrar
|
||||
if registrar, ok := globalRecorder.(PoolRegistrar); ok {
|
||||
// RegisterPools registers connection pools with the global recorder. pipelinePool
|
||||
// is the optional dedicated pipeline connection pool (nil when not configured);
|
||||
// it is registered as a regular pool under a "_pipeline" name suffix.
|
||||
func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, pipelinePool pool.Pooler, addr string) {
|
||||
// Check if the global recorder implements PoolRegistrar. Read it through
|
||||
// getRecorder: SetGlobalRecorder writes globalRecorder under recorderMu, and
|
||||
// clients are created (and closed) concurrently with telemetry being
|
||||
// installed, so an unlocked read here is a data race -race reports.
|
||||
if registrar, ok := getRecorder().(PoolRegistrar); ok {
|
||||
// Generate a unique ID for this client's pools
|
||||
uniqueID := generateUniqueID()
|
||||
|
||||
@@ -281,18 +295,27 @@ func RegisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, addr string) {
|
||||
poolName := addr + "_" + uniqueID + "_pubsub"
|
||||
registrar.RegisterPubSubPool(poolName, pubSubPool)
|
||||
}
|
||||
if pipelinePool != nil {
|
||||
poolName := addr + "_" + uniqueID + "_pipeline"
|
||||
registrar.RegisterPool(poolName, pipelinePool)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// UnregisterPools removes connection pools from the global recorder
|
||||
func UnregisterPools(connPool pool.Pooler, pubSubPool PubSubPooler) {
|
||||
// Check if the global recorder implements PoolRegistrar
|
||||
if registrar, ok := globalRecorder.(PoolRegistrar); ok {
|
||||
// UnregisterPools removes connection pools from the global recorder. pipelinePool
|
||||
// is the optional dedicated pipeline connection pool (nil when not configured).
|
||||
func UnregisterPools(connPool pool.Pooler, pubSubPool PubSubPooler, pipelinePool pool.Pooler) {
|
||||
// Check if the global recorder implements PoolRegistrar (see RegisterPools
|
||||
// for why this goes through getRecorder rather than reading directly).
|
||||
if registrar, ok := getRecorder().(PoolRegistrar); ok {
|
||||
if connPool != nil {
|
||||
registrar.UnregisterPool(connPool)
|
||||
}
|
||||
if pubSubPool != nil {
|
||||
registrar.UnregisterPubSubPool(pubSubPool)
|
||||
}
|
||||
if pipelinePool != nil {
|
||||
registrar.UnregisterPool(pipelinePool)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+235
-8
@@ -45,7 +45,7 @@ func GetCachedTimeNs() int64 {
|
||||
}
|
||||
|
||||
// Global atomic counter for connection IDs
|
||||
var connIDCounter uint64
|
||||
var connIDCounter atomic.Uint64
|
||||
|
||||
// HandoffState represents the atomic state for connection handoffs
|
||||
// This struct is stored atomically to prevent race conditions between
|
||||
@@ -63,7 +63,7 @@ type atomicNetConn struct {
|
||||
|
||||
// generateConnID generates a fast unique identifier for a connection with zero allocations
|
||||
func generateConnID() uint64 {
|
||||
return atomic.AddUint64(&connIDCounter, 1)
|
||||
return connIDCounter.Add(1)
|
||||
}
|
||||
|
||||
type Conn struct {
|
||||
@@ -109,6 +109,18 @@ type Conn struct {
|
||||
expiresAt time.Time
|
||||
poolName string // Name of the pool this connection belongs to (for metrics)
|
||||
|
||||
// preparedFieldsets tracks HIMPORT fieldsets prepared on this
|
||||
// connection's current server session: fieldset name -> client-side
|
||||
// registry version. The server drops fieldsets when the session ends,
|
||||
// so the map is cleared whenever the underlying network connection is
|
||||
// replaced. preparedFieldsetsEpoch records the registry's discard-all
|
||||
// epoch the session was prepared under; a session behind the current
|
||||
// epoch replays HIMPORT DISCARDALL before its next HIMPORT command.
|
||||
// Guarded by preparedFieldsetsMu; the map is nil until first use.
|
||||
preparedFieldsetsMu sync.Mutex
|
||||
preparedFieldsets map[string]uint64
|
||||
preparedFieldsetsEpoch uint64
|
||||
|
||||
// When a goroutine closes a connection, it usually knows the reason, so closeReason is not needed.
|
||||
// closeReason is only used when an in-use connection is closed by another goroutine,
|
||||
// to inform the goroutine using the connection why the connection was closed.
|
||||
@@ -135,6 +147,24 @@ type Conn struct {
|
||||
initConnFunc func(context.Context, *Conn) error
|
||||
|
||||
onClose func() error
|
||||
|
||||
// onCscClose is the client-side-caching close hook, kept separate from
|
||||
// onClose (streaming-credentials cleanup) so neither clobbers the other.
|
||||
// Both keep overwrite semantics, so re-running initConn can't accumulate them.
|
||||
onCscClose func() error
|
||||
|
||||
// onCscReinit runs after the connection is claimed for reinitialization but
|
||||
// before its socket is replaced. CSC uses it to invalidate entries whose
|
||||
// server-side tracking coverage belongs to the old socket.
|
||||
onCscReinit func()
|
||||
|
||||
// cscReadPending requests one conservative drain after a command read through
|
||||
// a transport whose buffered state cannot be fully observed by MaybeHasData.
|
||||
cscReadPending atomic.Bool
|
||||
|
||||
// lastCscPeriodicProbeNs throttles bounded fallback reads on platforms and
|
||||
// opaque transports without a non-consuming readiness mechanism.
|
||||
lastCscPeriodicProbeNs atomic.Int64
|
||||
}
|
||||
|
||||
func NewConn(netConn net.Conn) *Conn {
|
||||
@@ -638,6 +668,18 @@ func (cn *Conn) SetOnClose(fn func() error) {
|
||||
cn.onClose = fn
|
||||
}
|
||||
|
||||
// SetOnCscClose sets the client-side-caching close hook, overwriting any
|
||||
// previous one. It runs on Close in addition to the SetOnClose callback.
|
||||
func (cn *Conn) SetOnCscClose(fn func() error) {
|
||||
cn.onCscClose = fn
|
||||
}
|
||||
|
||||
// SetOnCscReinit sets the client-side-caching pre-reinitialization hook,
|
||||
// overwriting any previous one.
|
||||
func (cn *Conn) SetOnCscReinit(fn func()) {
|
||||
cn.onCscReinit = fn
|
||||
}
|
||||
|
||||
// SetInitConnFunc sets the connection initialization function to be called on reconnections.
|
||||
func (cn *Conn) SetInitConnFunc(fn func(context.Context, *Conn) error) {
|
||||
cn.initConnFunc = fn
|
||||
@@ -661,6 +703,85 @@ func (cn *Conn) SetNetConn(netConn net.Conn) {
|
||||
cn.readerMu.Unlock()
|
||||
|
||||
cn.bw.Reset(netConn)
|
||||
|
||||
// A new socket is a new server session with no HIMPORT fieldsets and
|
||||
// nothing left to discard.
|
||||
cn.ClearPreparedFieldsets(0)
|
||||
}
|
||||
|
||||
// FieldsetPreparedVersion returns the client-side registry version at which
|
||||
// the named HIMPORT fieldset was prepared on this connection's current server
|
||||
// session, or 0 if it was not prepared on it (registry versions start at 1).
|
||||
func (cn *Conn) FieldsetPreparedVersion(name string) uint64 {
|
||||
cn.preparedFieldsetsMu.Lock()
|
||||
version := cn.preparedFieldsets[name]
|
||||
cn.preparedFieldsetsMu.Unlock()
|
||||
return version
|
||||
}
|
||||
|
||||
// MarkFieldsetPrepared records that the named HIMPORT fieldset was prepared
|
||||
// on this connection's current server session at the given registry version.
|
||||
// A session acquiring its first fieldset adopts the given discard-all epoch
|
||||
// (fieldsets prepared after an HIMPORT DISCARDALL are not subject to it);
|
||||
// the epoch never moves backwards, so a mark carrying an older snapshot
|
||||
// cannot regress a session already wiped at a newer epoch.
|
||||
func (cn *Conn) MarkFieldsetPrepared(name string, version, epoch uint64) {
|
||||
cn.preparedFieldsetsMu.Lock()
|
||||
if len(cn.preparedFieldsets) == 0 {
|
||||
cn.preparedFieldsets = make(map[string]uint64)
|
||||
if epoch > cn.preparedFieldsetsEpoch {
|
||||
cn.preparedFieldsetsEpoch = epoch
|
||||
}
|
||||
}
|
||||
cn.preparedFieldsets[name] = version
|
||||
cn.preparedFieldsetsMu.Unlock()
|
||||
}
|
||||
|
||||
// UnmarkFieldsetPrepared forgets that the named HIMPORT fieldset was prepared
|
||||
// on this connection, forcing a replay before the next HIMPORT SET using it.
|
||||
func (cn *Conn) UnmarkFieldsetPrepared(name string) {
|
||||
cn.preparedFieldsetsMu.Lock()
|
||||
delete(cn.preparedFieldsets, name)
|
||||
cn.preparedFieldsetsMu.Unlock()
|
||||
}
|
||||
|
||||
// HasPreparedFieldsets reports whether any HIMPORT fieldset is prepared on
|
||||
// this connection's current server session.
|
||||
func (cn *Conn) HasPreparedFieldsets() bool {
|
||||
cn.preparedFieldsetsMu.Lock()
|
||||
n := len(cn.preparedFieldsets)
|
||||
cn.preparedFieldsetsMu.Unlock()
|
||||
return n > 0
|
||||
}
|
||||
|
||||
// PreparedFieldsetNames returns the names of the HIMPORT fieldsets prepared
|
||||
// on this connection's current server session.
|
||||
func (cn *Conn) PreparedFieldsetNames() []string {
|
||||
cn.preparedFieldsetsMu.Lock()
|
||||
names := make([]string, 0, len(cn.preparedFieldsets))
|
||||
for name := range cn.preparedFieldsets {
|
||||
names = append(names, name)
|
||||
}
|
||||
cn.preparedFieldsetsMu.Unlock()
|
||||
return names
|
||||
}
|
||||
|
||||
// FieldsetEpoch returns the discard-all epoch this connection's prepared
|
||||
// fieldsets belong to (0 when none were ever prepared on the session).
|
||||
func (cn *Conn) FieldsetEpoch() uint64 {
|
||||
cn.preparedFieldsetsMu.Lock()
|
||||
epoch := cn.preparedFieldsetsEpoch
|
||||
cn.preparedFieldsetsMu.Unlock()
|
||||
return epoch
|
||||
}
|
||||
|
||||
// ClearPreparedFieldsets forgets all HIMPORT fieldsets prepared on this
|
||||
// connection and records the discard-all epoch the wipe corresponds to.
|
||||
func (cn *Conn) ClearPreparedFieldsets(epoch uint64) {
|
||||
cn.preparedFieldsetsMu.Lock()
|
||||
cn.preparedFieldsets = nil
|
||||
cn.preparedFieldsetsEpoch = epoch
|
||||
cn.preparedFieldsetsMu.Unlock()
|
||||
}
|
||||
|
||||
// GetNetConn safely returns the current network connection using atomic load (lock-free).
|
||||
@@ -694,6 +815,10 @@ func (cn *Conn) SetNetConnAndInitConn(ctx context.Context, netConn net.Conn) err
|
||||
return fmt.Errorf("cannot initialize connection from state %s: %w", finalState, err)
|
||||
}
|
||||
|
||||
if cn.onCscReinit != nil {
|
||||
cn.onCscReinit()
|
||||
}
|
||||
|
||||
// Replace the underlying connection
|
||||
cn.SetNetConn(netConn)
|
||||
|
||||
@@ -774,8 +899,13 @@ func (cn *Conn) MarkQueuedForHandoff() error {
|
||||
// Already unusable - this is fine, keep the new handoff state
|
||||
return nil
|
||||
}
|
||||
// Restore the original state if transition fails for other reasons
|
||||
cn.handoffStateAtomic.Store(currentState)
|
||||
// Restore the original handoff state only if nothing else changed it
|
||||
// since our CAS above. A concurrent handoff worker may have completed
|
||||
// the handoff and run ClearHandoffState in this window; a plain Store
|
||||
// would clobber that, resurrecting ShouldHandoff=true and wedging the
|
||||
// connection so it can never be acquired again. The CAS leaves the
|
||||
// worker's state intact when it has taken over.
|
||||
cn.handoffStateAtomic.CompareAndSwap(newState, currentState)
|
||||
return fmt.Errorf("failed to mark connection as unusable: %w", err)
|
||||
}
|
||||
return nil
|
||||
@@ -871,6 +1001,18 @@ func (cn *Conn) PeekReplyTypeSafe() (byte, error) {
|
||||
return cn.rd.PeekReplyType()
|
||||
}
|
||||
|
||||
// PeekReplyTypeForCheck peeks at the reply type while holding readerMu, so it is
|
||||
// safe against a concurrent SetNetConn resetting the reader during handoff.
|
||||
// Unlike PeekReplyTypeSafe it does not require the data to already be buffered:
|
||||
// the pool health check calls it after connCheck reports unexpected socket data,
|
||||
// and connCheck only MSG_PEEKs, so the byte still has to be pulled from the
|
||||
// socket into the reader here.
|
||||
func (cn *Conn) PeekReplyTypeForCheck() (byte, error) {
|
||||
cn.readerMu.RLock()
|
||||
defer cn.readerMu.RUnlock()
|
||||
return cn.rd.PeekReplyType()
|
||||
}
|
||||
|
||||
func (cn *Conn) Write(b []byte) (int, error) {
|
||||
// Lock-free netConn access for better performance
|
||||
if netConn := cn.getNetConn(); netConn != nil {
|
||||
@@ -907,6 +1049,29 @@ func (cn *Conn) WithReader(
|
||||
return fn(cn.rd)
|
||||
}
|
||||
|
||||
// WithReaderHardDeadline runs fn under a HARD read deadline of now+timeout,
|
||||
// bypassing getEffectiveReadTimeout so a relaxed maintenance timeout can't extend
|
||||
// it (used by the CSC drainer). Takes no context: an expired cycle ctx must not
|
||||
// become the socket deadline, or the read surfaces context.DeadlineExceeded, which
|
||||
// isBadConn treats as fatal.
|
||||
func (cn *Conn) WithReaderHardDeadline(
|
||||
timeout time.Duration, fn func(rd *proto.Reader) error,
|
||||
) (err error) {
|
||||
netConn := cn.getNetConn()
|
||||
if netConn == nil {
|
||||
return errConnectionNotAvailable
|
||||
}
|
||||
if err := netConn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if clearErr := netConn.SetReadDeadline(time.Time{}); clearErr != nil {
|
||||
err = clearErr
|
||||
}
|
||||
}()
|
||||
return fn(cn.rd)
|
||||
}
|
||||
|
||||
func (cn *Conn) WithWriter(
|
||||
ctx context.Context, timeout time.Duration, fn func(wr *proto.Writer) error,
|
||||
) error {
|
||||
@@ -944,17 +1109,29 @@ func (cn *Conn) IsClosed() bool {
|
||||
}
|
||||
|
||||
func (cn *Conn) Close() error {
|
||||
if cn.IsClosed() {
|
||||
return nil
|
||||
for {
|
||||
state := cn.stateMachine.GetState()
|
||||
if state == StateClosed {
|
||||
return nil
|
||||
}
|
||||
if cn.stateMachine.TryTransitionFast(state, StateClosed) {
|
||||
// TryTransitionFast deliberately skips waiter notification; Close
|
||||
// still needs to wake any goroutine waiting on initialization.
|
||||
cn.stateMachine.notifyWaiters()
|
||||
break
|
||||
}
|
||||
}
|
||||
// Transition to CLOSED state
|
||||
cn.stateMachine.Transition(StateClosed)
|
||||
|
||||
if cn.onClose != nil {
|
||||
// ignore error
|
||||
_ = cn.onClose()
|
||||
cn.onClose = nil
|
||||
}
|
||||
if cn.onCscClose != nil {
|
||||
// ignore error
|
||||
_ = cn.onCscClose()
|
||||
cn.onCscClose = nil
|
||||
}
|
||||
|
||||
// Lock-free netConn access for better performance
|
||||
if netConn := cn.getNetConn(); netConn != nil {
|
||||
@@ -974,6 +1151,56 @@ func (cn *Conn) MaybeHasData() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// CheckForData reports whether the socket has data ready and surfaces a
|
||||
// detected closed or failed socket.
|
||||
func (cn *Conn) CheckForData() (bool, error) {
|
||||
if netConn := cn.getNetConn(); netConn != nil {
|
||||
return checkForData(netConn)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// MarkCscReadPending requests one conservative CSC drain after a command read
|
||||
// when the transport may retain data that MaybeHasData cannot observe.
|
||||
func (cn *Conn) MarkCscReadPending() {
|
||||
netConn := cn.getNetConn()
|
||||
if netConn == nil {
|
||||
return
|
||||
}
|
||||
if needsCscReadProbe(netConn) {
|
||||
cn.cscReadPending.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
// TakeCscReadPending consumes the post-command conservative-drain request.
|
||||
func (cn *Conn) TakeCscReadPending() bool {
|
||||
return cn.cscReadPending.Swap(false)
|
||||
}
|
||||
|
||||
// TakeCscPeriodicReadPending schedules a throttled conservative read for
|
||||
// transports with no readiness mechanism. It returns true at most once per
|
||||
// interval, including when several drainer passes race.
|
||||
func (cn *Conn) TakeCscPeriodicReadPending(interval time.Duration) bool {
|
||||
netConn := cn.getNetConn()
|
||||
if netConn == nil || interval <= 0 || !needsCscPeriodicProbe(netConn) {
|
||||
return false
|
||||
}
|
||||
|
||||
now := time.Since(cn.createdAt).Nanoseconds()
|
||||
if now <= 0 {
|
||||
now = 1
|
||||
}
|
||||
for {
|
||||
last := cn.lastCscPeriodicProbeNs.Load()
|
||||
if last != 0 && now >= last && now-last < int64(interval) {
|
||||
return false
|
||||
}
|
||||
if cn.lastCscPeriodicProbeNs.CompareAndSwap(last, now) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// deadline computes the effective deadline time based on context and timeout.
|
||||
// It updates the usedAt timestamp to now.
|
||||
// Uses cached time to avoid expensive syscall (max 50ms staleness is acceptable for deadline calculation).
|
||||
|
||||
+65
-1
@@ -19,10 +19,19 @@ func connCheck(conn net.Conn) error {
|
||||
// Reset previous timeout.
|
||||
_ = conn.SetDeadline(time.Time{})
|
||||
|
||||
// Health checks deliberately inspect only the outer connection. Unwrapping a
|
||||
// buffered transport such as crypto/tls.Conn can reveal an encrypted
|
||||
// post-handshake record and make isHealthyConn call PeekReplyType on the TLS
|
||||
// stream. With the deadline cleared above, TLS may consume that control record
|
||||
// and then wait forever for application data.
|
||||
sysConn, ok := conn.(syscall.Conn)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return checkSyscallConn(sysConn)
|
||||
}
|
||||
|
||||
func checkSyscallConn(sysConn syscall.Conn) error {
|
||||
rawConn, err := sysConn.SyscallConn()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -53,7 +62,62 @@ func connCheck(conn net.Conn) error {
|
||||
return sysErr
|
||||
}
|
||||
|
||||
// underlyingSyscallConn unwraps connections that expose their transport through
|
||||
// NetConn (notably crypto/tls.Conn). Limit the walk so a broken wrapper cannot
|
||||
// loop forever.
|
||||
func underlyingSyscallConn(conn net.Conn) (syscall.Conn, bool) {
|
||||
for range 8 {
|
||||
if sysConn, ok := conn.(syscall.Conn); ok {
|
||||
return sysConn, true
|
||||
}
|
||||
unwrapper, ok := conn.(interface{ NetConn() net.Conn })
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
conn = unwrapper.NetConn()
|
||||
if conn == nil {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// maybeHasData checks if there is data in the socket without consuming it
|
||||
func maybeHasData(conn net.Conn) bool {
|
||||
return connCheck(conn) == errUnexpectedRead
|
||||
hasData, _ := checkForData(conn)
|
||||
return hasData
|
||||
}
|
||||
|
||||
func checkForData(conn net.Conn) (bool, error) {
|
||||
// Unlike the general health check, CSC only uses this as a readiness hint
|
||||
// before a bounded read, so unwrapping TLS is safe and avoids blocking.
|
||||
_ = conn.SetDeadline(time.Time{})
|
||||
sysConn, ok := underlyingSyscallConn(conn)
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
switch err := checkSyscallConn(sysConn); err {
|
||||
case nil:
|
||||
return false, nil
|
||||
case errUnexpectedRead:
|
||||
return true, nil
|
||||
default:
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
// needsCscReadProbe reports whether a command read may leave data hidden from
|
||||
// maybeHasData. On Unix a direct syscall.Conn has no intermediate buffering;
|
||||
// TLS and opaque wrappers need one bounded post-command probe.
|
||||
func needsCscReadProbe(conn net.Conn) bool {
|
||||
_, direct := conn.(syscall.Conn)
|
||||
return !direct
|
||||
}
|
||||
|
||||
// needsCscPeriodicProbe reports whether the platform can inspect the transport
|
||||
// at all. Opaque wrappers get a throttled bounded fallback so invalidations that
|
||||
// arrive after the post-command probe are still eventually consumed.
|
||||
func needsCscPeriodicProbe(conn net.Conn) bool {
|
||||
_, ok := underlyingSyscallConn(conn)
|
||||
return !ok
|
||||
}
|
||||
|
||||
+15
-1
@@ -14,7 +14,21 @@ func connCheck(_ net.Conn) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// since we can't check for data on the socket, we just assume there is some
|
||||
// There is no portable non-consuming readiness check on this platform.
|
||||
// Returning true would force every idle CSC connection through a timed read on
|
||||
// every drainer tick. The CSC drainer uses needsCscPeriodicProbe instead.
|
||||
func maybeHasData(_ net.Conn) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func checkForData(_ net.Conn) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func needsCscReadProbe(_ net.Conn) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func needsCscPeriodicProbe(_ net.Conn) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
+302
-35
@@ -72,6 +72,11 @@ var (
|
||||
|
||||
// errConnNotPooled is returned when trying to return a non-pooled connection to the pool.
|
||||
errConnNotPooled = errors.New("connection not pooled")
|
||||
|
||||
// errConnEvictedIdle is passed to OnRemove hooks when a pooled connection is evicted on
|
||||
// Put because the idle pool is already at MaxIdleConns.
|
||||
errConnEvictedIdle = errors.New("connection evicted: idle pool at capacity")
|
||||
|
||||
// metricCallbackMu protects all global metric callback functions for thread-safe access.
|
||||
metricCallbackMu sync.RWMutex
|
||||
|
||||
@@ -304,6 +309,9 @@ func getMetricPendingRequestsCallback() func(ctx context.Context, delta int, cn
|
||||
}
|
||||
|
||||
// Stats contains pool state information and accumulated stats.
|
||||
//
|
||||
// TODO(cxl): the uint32/int64 fields below will be changed to atomic value
|
||||
// types (atomic.Uint32/atomic.Int64) in v10, which is a breaking API change.
|
||||
type Stats struct {
|
||||
Hits uint32 // number of times free connection was found in the pool
|
||||
Misses uint32 // number of times free connection was NOT found in the pool
|
||||
@@ -318,6 +326,11 @@ type Stats struct {
|
||||
PendingRequests uint32 // number of pending requests waiting for a connection
|
||||
|
||||
PubSubStats PubSubStats
|
||||
|
||||
// PipelineStats holds the stats of the separate pipeline connection pool
|
||||
// when one is configured (PipelineReadBufferSize/PipelineWriteBufferSize).
|
||||
// nil when pipelines share the main pool.
|
||||
PipelineStats *Stats
|
||||
}
|
||||
|
||||
type ConnRetirer interface {
|
||||
@@ -394,7 +407,7 @@ type lastDialErrorWrap struct {
|
||||
type ConnPool struct {
|
||||
cfg *Options
|
||||
|
||||
dialErrorsNum uint32 // atomic
|
||||
dialErrorsNum atomic.Uint32
|
||||
lastDialError atomic.Value
|
||||
|
||||
dialsInProgress chan struct{}
|
||||
@@ -415,11 +428,23 @@ type ConnPool struct {
|
||||
stats Stats
|
||||
waitDurationNs atomic.Int64
|
||||
|
||||
_closed uint32 // atomic
|
||||
_closed atomic.Uint32
|
||||
|
||||
// Pool hooks manager for flexible connection processing
|
||||
// Using atomic.Pointer for lock-free reads in hot paths (Get/Put)
|
||||
// Pool hooks manager. atomic.Pointer keeps hot-path reads (Get/Put)
|
||||
// lock-free; hookMu serializes Add/RemovePoolHook's read-clone-store so
|
||||
// concurrent mutators (e.g. maintnotifications and CSC) can't lose an update.
|
||||
hookManager atomic.Pointer[PoolHookManager]
|
||||
hookMu sync.Mutex
|
||||
|
||||
// drainMu/drainDone coordinate the CSC drainer's temporary idle-connection
|
||||
// claim with Get. The normal semaphore retains its PoolSize capacity (and
|
||||
// therefore the established MaxActiveConns/ErrPoolExhausted behavior); a Get
|
||||
// that finds the idle list empty only because the drainer borrowed a conn
|
||||
// waits for that short claim to finish instead of opening an overflow conn.
|
||||
drainMu sync.Mutex
|
||||
drainDone chan struct{}
|
||||
drainBorrowed int
|
||||
drainGeneration atomic.Uint64
|
||||
}
|
||||
|
||||
var _ Pooler = (*ConnPool)(nil)
|
||||
@@ -453,7 +478,10 @@ func (p *ConnPool) initializeHooks() {
|
||||
|
||||
// AddPoolHook adds a pool hook to the pool.
|
||||
func (p *ConnPool) AddPoolHook(hook PoolHook) {
|
||||
// Lock-free read of current manager
|
||||
// Serialize so a concurrent Add/Remove can't clobber this change.
|
||||
p.hookMu.Lock()
|
||||
defer p.hookMu.Unlock()
|
||||
|
||||
manager := p.hookManager.Load()
|
||||
if manager == nil {
|
||||
p.initializeHooks()
|
||||
@@ -464,12 +492,22 @@ func (p *ConnPool) AddPoolHook(hook PoolHook) {
|
||||
newManager := manager.Clone()
|
||||
newManager.AddHook(hook)
|
||||
|
||||
// Atomically swap to new manager
|
||||
// Atomically swap to new manager (hot-path readers load lock-free)
|
||||
p.hookManager.Store(newManager)
|
||||
}
|
||||
|
||||
// SupportsPoolHooks reports that AddPoolHook and RemovePoolHook are functional.
|
||||
// Pooler adapters with no-op hook methods intentionally do not expose this
|
||||
// optional capability.
|
||||
func (p *ConnPool) SupportsPoolHooks() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// RemovePoolHook removes a pool hook from the pool.
|
||||
func (p *ConnPool) RemovePoolHook(hook PoolHook) {
|
||||
p.hookMu.Lock()
|
||||
defer p.hookMu.Unlock()
|
||||
|
||||
manager := p.hookManager.Load()
|
||||
if manager != nil {
|
||||
// Create new manager with removed hook
|
||||
@@ -651,7 +689,7 @@ func (p *ConnPool) dialConn(ctx context.Context, pooled bool) (*Conn, error) {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
|
||||
if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.cfg.PoolSize) {
|
||||
if p.dialErrorsNum.Load() >= uint32(p.cfg.PoolSize) {
|
||||
return nil, p.getLastDialError()
|
||||
}
|
||||
|
||||
@@ -724,7 +762,7 @@ func (p *ConnPool) dialConn(ctx context.Context, pooled bool) (*Conn, error) {
|
||||
internal.Logger.Printf(ctx, "redis: connection pool: failed to dial after %d attempts: %v", attempt, lastErr)
|
||||
// All retries failed - handle error tracking
|
||||
p.setLastDialError(lastErr)
|
||||
if atomic.AddUint32(&p.dialErrorsNum, 1) == uint32(p.cfg.PoolSize) {
|
||||
if p.dialErrorsNum.Add(1) == uint32(p.cfg.PoolSize) {
|
||||
go p.tryDial()
|
||||
}
|
||||
return nil, lastErr
|
||||
@@ -789,7 +827,7 @@ func (p *ConnPool) tryDial() {
|
||||
continue
|
||||
}
|
||||
|
||||
atomic.StoreUint32(&p.dialErrorsNum, 0)
|
||||
p.dialErrorsNum.Store(0)
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
@@ -835,28 +873,30 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) {
|
||||
cb(ctx, -1, nil, poolName)
|
||||
}
|
||||
}
|
||||
if err == ErrPoolTimeout {
|
||||
atomic.AddUint32(&p.stats.Timeouts, 1)
|
||||
if cb := getMetricConnectionTimeoutCallback(); cb != nil {
|
||||
cb(ctx, nil, "pool")
|
||||
}
|
||||
if cb := GetMetricErrorCallback(); cb != nil {
|
||||
cb(ctx, "POOL_TIMEOUT", nil, "POOL_TIMEOUT", true, 0)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Track wait time - only call time.Now() if callback is registered
|
||||
// PoolTimeout is one budget for both the pool turn and a drainer handoff.
|
||||
poolDeadline := time.Now().Add(p.cfg.PoolTimeout)
|
||||
|
||||
// Connection wait time measures only semaphore acquisition.
|
||||
var waitStart time.Time
|
||||
var waitDuration time.Duration
|
||||
waitTimeCallback := getMetricConnectionWaitTimeCallback()
|
||||
if waitTimeCallback != nil {
|
||||
waitStart = time.Now()
|
||||
}
|
||||
if err = p.waitTurn(ctx); err != nil {
|
||||
// Record timeout if applicable
|
||||
if err == ErrPoolTimeout {
|
||||
if cb := getMetricConnectionTimeoutCallback(); cb != nil {
|
||||
cb(ctx, nil, "pool")
|
||||
}
|
||||
// Record general error metric for pool timeout
|
||||
if cb := GetMetricErrorCallback(); cb != nil {
|
||||
cb(ctx, "POOL_TIMEOUT", nil, "POOL_TIMEOUT", true, 0)
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var waitDuration time.Duration
|
||||
if waitTimeCallback != nil {
|
||||
waitDuration = time.Since(waitStart)
|
||||
}
|
||||
@@ -867,6 +907,8 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) {
|
||||
// Lock-free atomic read - no mutex overhead!
|
||||
hookManager := p.hookManager.Load()
|
||||
|
||||
retryIdle:
|
||||
drainGeneration := p.drainGeneration.Load()
|
||||
for attempts := 0; attempts < getAttempts; attempts++ {
|
||||
|
||||
p.connsMu.Lock()
|
||||
@@ -937,6 +979,21 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) {
|
||||
return cn, nil
|
||||
}
|
||||
|
||||
// If the CSC drainer removed the only idle connection during this scan,
|
||||
// wait for that bounded maintenance claim and retry. The generation closes
|
||||
// the race where the drainer returns the connection between popIdle and this
|
||||
// check. Normal MaxActiveConns exhaustion still proceeds to newConn and
|
||||
// returns ErrPoolExhausted immediately, preserving the existing contract.
|
||||
if done, retry := p.drainerWaitState(drainGeneration); done != nil {
|
||||
if err = p.waitForDrainer(ctx, done, poolDeadline); err != nil {
|
||||
p.freeTurn()
|
||||
return nil, err
|
||||
}
|
||||
goto retryIdle
|
||||
} else if retry {
|
||||
goto retryIdle
|
||||
}
|
||||
|
||||
atomic.AddUint32(&p.stats.Misses, 1)
|
||||
|
||||
var newcn *Conn
|
||||
@@ -964,7 +1021,7 @@ func (p *ConnPool) getConn(ctx context.Context) (cn *Conn, err error) {
|
||||
// causing IsInited()=true. This means _getConn() in redis.go will take the
|
||||
// early return path and never reach its create time recording.
|
||||
// When hookManager is nil, _getConn() handles both initialization and create time recording.
|
||||
if dialStartNs := newcn.GetDialStartNs(); dialStartNs > 0 {
|
||||
if dialStartNs := newcn.GetDialStartNs(); newcn.IsInited() && dialStartNs > 0 {
|
||||
if cb := GetMetricConnectionCreateTimeCallback(); cb != nil {
|
||||
duration := time.Duration(time.Now().UnixNano() - dialStartNs)
|
||||
cb(ctx, duration, newcn)
|
||||
@@ -1120,23 +1177,83 @@ func (p *ConnPool) waitTurn(ctx context.Context) error {
|
||||
// Slow path: need to wait
|
||||
start := time.Now()
|
||||
err := p.semaphore.Acquire(ctx, p.cfg.PoolTimeout, ErrPoolTimeout)
|
||||
|
||||
switch err {
|
||||
case nil:
|
||||
// Successfully acquired after waiting
|
||||
p.waitDurationNs.Add(time.Now().UnixNano() - start.UnixNano())
|
||||
atomic.AddUint32(&p.stats.WaitCount, 1)
|
||||
case ErrPoolTimeout:
|
||||
atomic.AddUint32(&p.stats.Timeouts, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return err
|
||||
p.waitDurationNs.Add(time.Now().UnixNano() - start.UnixNano())
|
||||
atomic.AddUint32(&p.stats.WaitCount, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *ConnPool) freeTurn() {
|
||||
p.semaphore.Release()
|
||||
}
|
||||
|
||||
func (p *ConnPool) beginDrainerBorrow() {
|
||||
p.drainMu.Lock()
|
||||
if p.drainBorrowed == 0 {
|
||||
p.drainDone = make(chan struct{})
|
||||
}
|
||||
p.drainBorrowed++
|
||||
p.drainMu.Unlock()
|
||||
}
|
||||
|
||||
func (p *ConnPool) endDrainerBorrow() {
|
||||
p.drainMu.Lock()
|
||||
p.drainBorrowed--
|
||||
if p.drainBorrowed == 0 {
|
||||
close(p.drainDone)
|
||||
p.drainDone = nil
|
||||
p.drainGeneration.Add(1)
|
||||
}
|
||||
p.drainMu.Unlock()
|
||||
}
|
||||
|
||||
// drainerWaitState returns the current drain epoch's completion channel. If no
|
||||
// drain is active, retry reports whether an epoch completed during the caller's
|
||||
// idle scan and the idle list therefore needs to be checked again.
|
||||
func (p *ConnPool) drainerWaitState(generation uint64) (done <-chan struct{}, retry bool) {
|
||||
p.drainMu.Lock()
|
||||
defer p.drainMu.Unlock()
|
||||
if p.drainBorrowed > 0 {
|
||||
return p.drainDone, false
|
||||
}
|
||||
return nil, p.drainGeneration.Load() != generation
|
||||
}
|
||||
|
||||
func (p *ConnPool) waitForDrainer(
|
||||
ctx context.Context, done <-chan struct{}, poolDeadline time.Time,
|
||||
) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
remaining := time.Until(poolDeadline)
|
||||
if remaining <= 0 {
|
||||
return ErrPoolTimeout
|
||||
}
|
||||
timer := time.NewTimer(remaining)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
// Prefer a caller cancellation that raced with the pool timeout.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return ErrPoolTimeout
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ConnPool) popIdle() (*Conn, error) {
|
||||
if p.closed() {
|
||||
return nil, ErrClosed
|
||||
@@ -1290,6 +1407,9 @@ func (p *ConnPool) putConn(ctx context.Context, cn *Conn, freeTurn bool) {
|
||||
// expected state, don't log it
|
||||
case StateClosed:
|
||||
internal.Logger.Printf(ctx, "Unexpected conn[%d] state changed by hook to %v, closing it", cn.GetID(), currentState)
|
||||
if hookManager != nil {
|
||||
hookManager.ProcessOnRemove(ctx, cn, errHookRequestedRemoval)
|
||||
}
|
||||
shouldCloseConn = true
|
||||
removedFromPool = p.removeConnWithLock(cn)
|
||||
default:
|
||||
@@ -1357,6 +1477,9 @@ func (p *ConnPool) putConn(ctx context.Context, cn *Conn, freeTurn bool) {
|
||||
}
|
||||
} else {
|
||||
shouldCloseConn = true
|
||||
if hookManager != nil {
|
||||
hookManager.ProcessOnRemove(ctx, cn, errConnEvictedIdle)
|
||||
}
|
||||
removedFromPool = p.removeConnWithLock(cn)
|
||||
|
||||
// Only emit if we actually removed it from the map (not already taken by Close()).
|
||||
@@ -1548,6 +1671,13 @@ func (p *ConnPool) IdleLen() int {
|
||||
return int(n)
|
||||
}
|
||||
|
||||
// Name returns the pool's configured name, which is stamped on every
|
||||
// connection it creates (Conn.PoolName). Callers holding a Pooler can type
|
||||
// assert to interface{ Name() string } to find which pool owns a connection —
|
||||
// used by maintnotifications to route a handoff to the hook that owns the
|
||||
// conn's pool rather than always the primary one.
|
||||
func (p *ConnPool) Name() string { return p.cfg.Name }
|
||||
|
||||
// Size returns the maximum pool size (capacity).
|
||||
//
|
||||
// This is used by the streaming credentials manager to size the re-auth worker pool,
|
||||
@@ -1573,7 +1703,7 @@ func (p *ConnPool) Stats() *Stats {
|
||||
}
|
||||
|
||||
func (p *ConnPool) closed() bool {
|
||||
return atomic.LoadUint32(&p._closed) == 1
|
||||
return p._closed.Load() == 1
|
||||
}
|
||||
|
||||
func (p *ConnPool) RetireConns(ctx context.Context, conns []*Conn, reason string) {
|
||||
@@ -1652,8 +1782,142 @@ func (p *ConnPool) Filter(fn func(*Conn) bool) error {
|
||||
return firstErr
|
||||
}
|
||||
|
||||
type drainConn struct {
|
||||
conn *Conn
|
||||
idleIndex int
|
||||
}
|
||||
|
||||
// DrainState carries the cross-pass round bookkeeping for the CSC drainer.
|
||||
// It is owned by one drainer goroutine, so no synchronization is needed.
|
||||
type DrainState struct {
|
||||
// round contains only initialized idle connections. Entries are processed
|
||||
// from the end so their snapshot indexes remain stable as connections are
|
||||
// removed and returned. Connections that go idle mid-round are deferred.
|
||||
round []drainConn
|
||||
next int
|
||||
}
|
||||
|
||||
// DrainIdleConns runs one pass of the CSC invalidation drainer over the current
|
||||
// round, holding AT MOST ONE connection and its pool turn at a time (ctx is the
|
||||
// per-cycle deadline). A round = idle conn ids snapshotted at start; mid-round
|
||||
// arrivals are deferred. Each member is drainerPop'd and drained by fn, or — if no
|
||||
// longer a claimable idle conn — reconciled (marked visited) so it can't hang the
|
||||
// round. The drainer yields when no turn is immediately available, giving command
|
||||
// traffic priority. Handles at least one member before honoring ctx (so a tiny
|
||||
// DrainInterval can't stall it). No-ops if the pool is closed.
|
||||
func (p *ConnPool) DrainIdleConns(ctx context.Context, st *DrainState, fn func(cn *Conn) error) {
|
||||
if st == nil || fn == nil || p.closed() {
|
||||
return
|
||||
}
|
||||
|
||||
if st.round == nil {
|
||||
st.round = p.idleConnsSnapshot()
|
||||
st.next = len(st.round)
|
||||
if len(st.round) == 0 {
|
||||
st.round = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
handled := 0
|
||||
for st.next > 0 {
|
||||
// Min-progress: handle at least one member — drained OR reconciled — before
|
||||
// honoring the per-cycle deadline, so a pass does a bounded amount of work
|
||||
// while ignoring an expired ctx. A deadline-truncated round resumes on the
|
||||
// next pass.
|
||||
if handled > 0 && ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Account for the borrowed connection exactly like Get. Without a turn,
|
||||
// a concurrent Get can observe the temporarily-empty idle pool and either
|
||||
// exceed PoolSize or fail at MaxActiveConns. Maintenance never waits for a
|
||||
// turn, so command traffic wins under contention.
|
||||
if !p.semaphore.TryAcquire() {
|
||||
return
|
||||
}
|
||||
st.next--
|
||||
cn := p.drainerPop(ctx, st.round[st.next])
|
||||
if cn == nil {
|
||||
p.freeTurn()
|
||||
// Reconcile: not a claimable idle member right now (closed, in use,
|
||||
// unusable, or moved in idleConns by concurrent traffic). Covered by
|
||||
// the command-path drain and/or the next round.
|
||||
handled++
|
||||
continue
|
||||
}
|
||||
|
||||
func() {
|
||||
defer p.endDrainerBorrow()
|
||||
if err := fn(cn); err != nil {
|
||||
// Fatal drain error (read/protocol/connection).
|
||||
p.removeConnInternal(ctx, cn, err, true)
|
||||
} else {
|
||||
// Normal return: runs OnPut (queues any maintenance handoff).
|
||||
p.putConn(ctx, cn, true)
|
||||
}
|
||||
}()
|
||||
handled++
|
||||
}
|
||||
|
||||
// Every member handled — round complete; snapshot a fresh round next pass.
|
||||
st.round = nil
|
||||
st.next = 0
|
||||
}
|
||||
|
||||
// idleConnsSnapshot returns initialized idle connections and their current
|
||||
// indexes. StateCreated MinIdleConns are intentionally excluded.
|
||||
func (p *ConnPool) idleConnsSnapshot() []drainConn {
|
||||
p.connsMu.Lock()
|
||||
defer p.connsMu.Unlock()
|
||||
if len(p.idleConns) == 0 {
|
||||
return nil
|
||||
}
|
||||
round := make([]drainConn, 0, len(p.idleConns))
|
||||
for idx, cn := range p.idleConns {
|
||||
if cn.stateMachine.GetState() == StateIdle {
|
||||
round = append(round, drainConn{conn: cn, idleIndex: idx})
|
||||
}
|
||||
}
|
||||
return round
|
||||
}
|
||||
|
||||
// drainerPop claims a snapshotted connection (strict IDLE->IN_USE) and removes
|
||||
// it from idleConns in O(1). Entries are processed in reverse index order, so
|
||||
// swap removal cannot move an unprocessed round member. If concurrent pool
|
||||
// traffic changed the slot, the member is deferred to the next round.
|
||||
func (p *ConnPool) drainerPop(ctx context.Context, member drainConn) *Conn {
|
||||
p.connsMu.Lock()
|
||||
defer p.connsMu.Unlock()
|
||||
if p.closed() {
|
||||
return nil
|
||||
}
|
||||
idx := member.idleIndex
|
||||
if idx < 0 || idx >= len(p.idleConns) || p.idleConns[idx] != member.conn {
|
||||
return nil
|
||||
}
|
||||
cn := member.conn
|
||||
if !cn.stateMachine.TryTransitionFast(StateIdle, StateInUse) {
|
||||
return nil
|
||||
}
|
||||
p.beginDrainerBorrow()
|
||||
last := len(p.idleConns) - 1
|
||||
p.idleConns[idx] = p.idleConns[last]
|
||||
p.idleConns[last] = nil
|
||||
p.idleConns = p.idleConns[:last]
|
||||
p.idleConnsLen.Add(-1)
|
||||
if cb := getMetricConnectionStateChangeCallback(); cb != nil {
|
||||
cb(ctx, cn, MetricStateIdle, MetricStateUsed)
|
||||
}
|
||||
if cb := getMetricConnectionCountCallback(); cb != nil {
|
||||
cb(ctx, -1, cn, "idle", false)
|
||||
cb(ctx, 1, cn, "used", false)
|
||||
}
|
||||
return cn
|
||||
}
|
||||
|
||||
func (p *ConnPool) Close() error {
|
||||
if !atomic.CompareAndSwapUint32(&p._closed, 0, 1) {
|
||||
if !p._closed.CompareAndSwap(0, 1) {
|
||||
return ErrClosed
|
||||
}
|
||||
|
||||
@@ -1735,8 +1999,11 @@ func (p *ConnPool) isHealthyConn(cn *Conn, nowNs int64) bool {
|
||||
if err := connCheck(cn.getNetConn()); err != nil {
|
||||
// If there's unexpected data, it might be push notifications (RESP3)
|
||||
if p.cfg.PushNotificationsEnabled && err == errUnexpectedRead {
|
||||
// Peek at the reply type to check if it's a push notification
|
||||
if replyType, err := cn.rd.PeekReplyType(); err == nil && replyType == proto.RespPush {
|
||||
// Peek at the reply type to check if it's a push notification.
|
||||
// Use the readerMu-guarded peek: a concurrent handoff may be
|
||||
// resetting cn.rd via SetNetConn on a connection popped by Get
|
||||
// before the OnGet state check rejects it.
|
||||
if replyType, err := cn.PeekReplyTypeForCheck(); err == nil && replyType == proto.RespPush {
|
||||
// For RESP3 connections with push notifications, we allow some buffered data
|
||||
// The client will process these notifications before using the connection
|
||||
internal.Logger.Printf(
|
||||
|
||||
+33
-11
@@ -35,11 +35,16 @@ func (e BadConnError) Unwrap() error {
|
||||
|
||||
type StickyConnPool struct {
|
||||
pool Pooler
|
||||
shared int32 // atomic
|
||||
shared atomic.Int32
|
||||
|
||||
state uint32 // atomic
|
||||
state atomic.Uint32
|
||||
ch chan *Conn
|
||||
|
||||
// onFirstConn runs once when this sticky pool claims a connection from its
|
||||
// parent. CSC uses it to revoke cache ownership before the connection leaves
|
||||
// the parent's background drainer.
|
||||
onFirstConn func(*Conn)
|
||||
|
||||
_badConnError atomic.Value
|
||||
}
|
||||
|
||||
@@ -53,7 +58,7 @@ func NewStickyConnPool(pool Pooler) *StickyConnPool {
|
||||
ch: make(chan *Conn, 1),
|
||||
}
|
||||
}
|
||||
atomic.AddInt32(&p.shared, 1)
|
||||
p.shared.Add(1)
|
||||
return p
|
||||
}
|
||||
|
||||
@@ -68,13 +73,16 @@ func (p *StickyConnPool) CloseConn(ctx context.Context, cn *Conn, reason string,
|
||||
func (p *StickyConnPool) Get(ctx context.Context) (*Conn, error) {
|
||||
// In worst case this races with Close which is not a very common operation.
|
||||
for i := 0; i < 1000; i++ {
|
||||
switch atomic.LoadUint32(&p.state) {
|
||||
switch p.state.Load() {
|
||||
case stateDefault:
|
||||
cn, err := p.pool.Get(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if atomic.CompareAndSwapUint32(&p.state, stateDefault, stateInited) {
|
||||
if p.state.CompareAndSwap(stateDefault, stateInited) {
|
||||
if p.onFirstConn != nil {
|
||||
p.onFirstConn(cn)
|
||||
}
|
||||
return cn, nil
|
||||
}
|
||||
p.pool.Remove(ctx, cn, ErrClosed)
|
||||
@@ -96,12 +104,26 @@ func (p *StickyConnPool) Get(ctx context.Context) (*Conn, error) {
|
||||
return nil, fmt.Errorf("redis: StickyConnPool.Get: infinite loop")
|
||||
}
|
||||
|
||||
// SetOnFirstConn configures a callback that runs when the sticky pool first
|
||||
// claims a parent connection. It must be called before the pool is used.
|
||||
func (p *StickyConnPool) SetOnFirstConn(fn func(*Conn)) {
|
||||
p.onFirstConn = fn
|
||||
}
|
||||
|
||||
func (p *StickyConnPool) Put(ctx context.Context, cn *Conn) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
p.freeConn(ctx, cn)
|
||||
}
|
||||
}()
|
||||
// A connection marked for removal on release (it may hold unread
|
||||
// replies) must not be served to the next Get: record it as a bad
|
||||
// connection — exactly like Remove — so Get refuses and the underlying
|
||||
// connection is removed from the parent pool when the sticky pool
|
||||
// unwinds (the parent's Put honors the same mark).
|
||||
if reason := cn.CloseOnPutReason(); reason != "" {
|
||||
p._badConnError.Store(BadConnError{wrapped: errors.New(reason)})
|
||||
}
|
||||
p.ch <- cn
|
||||
}
|
||||
|
||||
@@ -130,16 +152,16 @@ func (p *StickyConnPool) RemoveWithoutTurn(ctx context.Context, cn *Conn, reason
|
||||
}
|
||||
|
||||
func (p *StickyConnPool) Close() error {
|
||||
if shared := atomic.AddInt32(&p.shared, -1); shared > 0 {
|
||||
if shared := p.shared.Add(-1); shared > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
state := atomic.LoadUint32(&p.state)
|
||||
state := p.state.Load()
|
||||
if state == stateClosed {
|
||||
return ErrClosed
|
||||
}
|
||||
if atomic.CompareAndSwapUint32(&p.state, state, stateClosed) {
|
||||
if p.state.CompareAndSwap(state, stateClosed) {
|
||||
close(p.ch)
|
||||
cn, ok := <-p.ch
|
||||
if ok {
|
||||
@@ -168,8 +190,8 @@ func (p *StickyConnPool) Reset(ctx context.Context) error {
|
||||
return errors.New("redis: StickyConnPool does not have a Conn")
|
||||
}
|
||||
|
||||
if !atomic.CompareAndSwapUint32(&p.state, stateInited, stateDefault) {
|
||||
state := atomic.LoadUint32(&p.state)
|
||||
if !p.state.CompareAndSwap(stateInited, stateDefault) {
|
||||
state := p.state.Load()
|
||||
return fmt.Errorf("redis: invalid StickyConnPool state: %d", state)
|
||||
}
|
||||
|
||||
@@ -186,7 +208,7 @@ func (p *StickyConnPool) badConnError() error {
|
||||
}
|
||||
|
||||
func (p *StickyConnPool) Len() int {
|
||||
switch atomic.LoadUint32(&p.state) {
|
||||
switch p.state.Load() {
|
||||
case stateDefault:
|
||||
return 0
|
||||
case stateInited:
|
||||
|
||||
+4
@@ -7,6 +7,10 @@ import (
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// PubSubStats contains pub/sub connection pool stats.
|
||||
//
|
||||
// TODO(cxl): the uint32 fields below will be changed to atomic.Uint32 in v10,
|
||||
// which is a breaking API change.
|
||||
type PubSubStats struct {
|
||||
Created uint32
|
||||
Untracked uint32
|
||||
|
||||
+127
-48
@@ -76,6 +76,11 @@ func (r *Reader) Buffered() int {
|
||||
return r.rd.Buffered()
|
||||
}
|
||||
|
||||
// Size returns the size of the underlying read buffer.
|
||||
func (r *Reader) Size() int {
|
||||
return r.rd.Size()
|
||||
}
|
||||
|
||||
func (r *Reader) Peek(n int) ([]byte, error) {
|
||||
return r.rd.Peek(n)
|
||||
}
|
||||
@@ -100,18 +105,30 @@ func (r *Reader) PeekReplyType() (byte, error) {
|
||||
return b[0], nil
|
||||
}
|
||||
|
||||
// MinRESP3ReadBufferSize is the minimum buffer size used when RESP3 push
|
||||
// notifications must be inspected without consuming them.
|
||||
const MinRESP3ReadBufferSize = 128
|
||||
|
||||
// ErrPushNotificationNameTooLong is returned when the push header does not fit
|
||||
// in the bounded peek window. Callers should consume the frame with ReadReply.
|
||||
var ErrPushNotificationNameTooLong = errors.New("redis: push notification name exceeds peek window")
|
||||
|
||||
// PeekPushNotificationName returns the notification name of the next RESP3
|
||||
// push frame without consuming it. The caller is expected to have already
|
||||
// verified that the next reply is a push notification (e.g. via PeekReplyType
|
||||
// returning RespPush).
|
||||
//
|
||||
// To identify the name the method may block briefly reading more bytes from
|
||||
// the underlying connection. That is safe: once the push marker '>' has been
|
||||
// observed, the server is committed to sending the rest of the frame, so
|
||||
// fetching the next few header bytes does not race with anything the caller
|
||||
// could be waiting on. Blocking is preferred to a truncated peek, which would
|
||||
// silently misidentify the notification and cause the caller's ReadReply to
|
||||
// consume (and drop) the frame; see issue #3839.
|
||||
// To identify the name the method may block reading more bytes from the
|
||||
// underlying connection, but only ever waits for one byte beyond the valid
|
||||
// frame prefix it has already seen. That byte is guaranteed to arrive: an
|
||||
// incomplete prefix means the server is still committed to sending the rest
|
||||
// of the frame. Demanding any fixed amount instead can deadlock — a complete
|
||||
// frame such as a subscribe confirmation for a short channel name can be
|
||||
// smaller than the fixed window, and once it is buffered the server has
|
||||
// nothing more to send (issue #3935). Blocking for in-flight bytes is
|
||||
// preferred to a truncated peek, which would silently misidentify the
|
||||
// notification and cause the caller's ReadReply to consume (and drop) the
|
||||
// frame; see issue #3839.
|
||||
func (r *Reader) PeekPushNotificationName() (string, error) {
|
||||
c, err := r.rd.Peek(1)
|
||||
if err != nil {
|
||||
@@ -121,16 +138,18 @@ func (r *Reader) PeekPushNotificationName() (string, error) {
|
||||
return "", fmt.Errorf("redis: can't peek push notification name, next reply is not a push notification")
|
||||
}
|
||||
|
||||
// Start with a peek window that covers every Redis-defined notification
|
||||
// header (MOVING, MIGRATING, FAILED_OVER, message, pmessage, smessage,
|
||||
// subscribe, unsubscribe, ...). If a longer name is encountered, grow
|
||||
// the window up to maxPushHeaderPeek before giving up.
|
||||
const initialPeek = 36
|
||||
const maxPushHeaderPeek = 4096
|
||||
|
||||
peekSize := initialPeek
|
||||
for {
|
||||
buf, peekErr := r.rd.Peek(peekSize)
|
||||
// Parse from what is already buffered; this never blocks.
|
||||
avail := r.rd.Buffered()
|
||||
if avail > maxPushHeaderPeek {
|
||||
avail = maxPushHeaderPeek
|
||||
}
|
||||
buf, peekErr := r.rd.Peek(avail)
|
||||
if peekErr != nil {
|
||||
return "", peekErr
|
||||
}
|
||||
name, complete, parseErr := parsePushNotificationName(buf)
|
||||
if parseErr != nil {
|
||||
return "", parseErr
|
||||
@@ -138,17 +157,17 @@ func (r *Reader) PeekPushNotificationName() (string, error) {
|
||||
if complete {
|
||||
return name, nil
|
||||
}
|
||||
// Parser ran out of bytes. Surface a failed underlying read before
|
||||
// growing further; otherwise grow the peek window and retry.
|
||||
if peekErr != nil {
|
||||
return "", peekErr
|
||||
if avail >= maxPushHeaderPeek {
|
||||
return "", ErrPushNotificationNameTooLong
|
||||
}
|
||||
if peekSize >= maxPushHeaderPeek {
|
||||
return "", fmt.Errorf("redis: push notification header exceeds %d bytes", maxPushHeaderPeek)
|
||||
}
|
||||
peekSize *= 2
|
||||
if peekSize > maxPushHeaderPeek {
|
||||
peekSize = maxPushHeaderPeek
|
||||
// Valid but incomplete prefix: the rest of the frame is in flight.
|
||||
// Block for exactly one more byte — the read that delivers it picks
|
||||
// up whatever else has already arrived — then re-parse.
|
||||
if _, err := r.rd.Peek(avail + 1); err != nil {
|
||||
if errors.Is(err, bufio.ErrBufferFull) {
|
||||
return "", ErrPushNotificationNameTooLong
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -440,6 +459,16 @@ func (r *Reader) readMap(line []byte) (map[interface{}]interface{}, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Reject unhashable keys (arrays/maps) before they are used as a map
|
||||
// key, which would otherwise panic. This check must run before the
|
||||
// value is read so it also guards the Nil and RedisError paths below,
|
||||
// which write the key into the map and continue.
|
||||
switch k.(type) {
|
||||
case []interface{}, map[interface{}]interface{}:
|
||||
return nil, fmt.Errorf("redis: RESP3 map key must be a scalar type, got %T", k)
|
||||
}
|
||||
|
||||
v, err := r.ReadReply()
|
||||
if err != nil {
|
||||
if err == Nil {
|
||||
@@ -452,6 +481,7 @@ func (r *Reader) readMap(line []byte) (map[interface{}]interface{}, error) {
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m[k] = v
|
||||
}
|
||||
return m, nil
|
||||
@@ -591,10 +621,33 @@ func (r *Reader) ReadStringInto(buf []byte) (int, error) {
|
||||
// bufio.Reader.Read first drains its internal buffer, then for
|
||||
// remaining data larger than its buffer size reads directly from the
|
||||
// underlying reader (socket) — effectively zero-copy.
|
||||
//
|
||||
// Fast path: when the caller VISIBLY hands over room for the trailing
|
||||
// CRLF too (len(buf) >= n+2), read the payload and the CRLF in a
|
||||
// single io.ReadFull. For large values this is one direct socket read
|
||||
// instead of a big read followed by a tiny separate Discard(2) read,
|
||||
// which is what makes GetToBuffer beat a regular Get (no payload
|
||||
// allocation and the same number of reads). The 2 trailing bytes land
|
||||
// past the returned length and are ignored.
|
||||
//
|
||||
// The gate is on len, NOT cap: a sub-slice of a larger buffer (e.g.
|
||||
// packed segments big[i*slot:(i+1)*slot]) exposes trailing capacity
|
||||
// that belongs to the caller's NEXT segment — writing the CRLF there
|
||||
// would silently corrupt caller-owned memory outside the slice they
|
||||
// passed. Callers who want the fast path pass len == payload+2 (the
|
||||
// returned length is still the payload length).
|
||||
if len(buf) >= n+2 {
|
||||
full := buf[:n+2]
|
||||
if _, err := io.ReadFull(r.rd, full); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
// Slow path: buffer is exactly large enough for the payload only, so
|
||||
// read the payload into it and discard the CRLF separately.
|
||||
if _, err := io.ReadFull(r.rd, buf[:n]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Discard trailing \r\n.
|
||||
if _, err := r.rd.Discard(2); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -738,7 +791,15 @@ func (r *Reader) Discard(line []byte) (err error) {
|
||||
}
|
||||
|
||||
n, err := replyLen(line)
|
||||
if err != nil && err != Nil {
|
||||
if err != nil {
|
||||
if err == Nil {
|
||||
// A nil reply ($-1, =-1, !-1, *-1, %-1) carries no payload; the
|
||||
// header line was already consumed by readLine, so there is
|
||||
// nothing to discard. Falling through would Discard(n+2)==2 bytes
|
||||
// that belong to the next reply and desync the stream, matching
|
||||
// how readRawReplyBuf/readRawReplyWriteTo already treat Nil.
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -755,8 +816,14 @@ func (r *Reader) Discard(line []byte) (err error) {
|
||||
}
|
||||
return nil
|
||||
case RespMap, RespAttr:
|
||||
// Read key & value.
|
||||
for i := 0; i < n*2; i++ {
|
||||
// Iterate over the n key/value pairs rather than n*2 elements: a count
|
||||
// above MaxInt/2 makes n*2 overflow to a negative loop bound, which
|
||||
// would skip the body entirely and return nil, leaving the map bytes in
|
||||
// the stream for the next reply to consume (a silent desync).
|
||||
for i := 0; i < n; i++ {
|
||||
if err = r.DiscardNext(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = r.DiscardNext(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -849,10 +916,12 @@ func (r *Reader) readRawReplyBuf(buf []byte) ([]byte, error) {
|
||||
}
|
||||
return buf, err
|
||||
}
|
||||
for i := 0; i < n*2; i++ {
|
||||
buf, err = r.readRawReplyBuf(buf)
|
||||
if err != nil {
|
||||
return buf, err
|
||||
for i := 0; i < n; i++ {
|
||||
for pair := 0; pair < 2; pair++ {
|
||||
buf, err = r.readRawReplyBuf(buf)
|
||||
if err != nil {
|
||||
return buf, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return buf, nil
|
||||
@@ -867,11 +936,15 @@ func (r *Reader) readRawReplyBuf(buf []byte) ([]byte, error) {
|
||||
}
|
||||
return buf, err
|
||||
}
|
||||
// Read the attribute key-value pairs
|
||||
for i := 0; i < n*2; i++ {
|
||||
buf, err = r.readRawReplyBuf(buf)
|
||||
if err != nil {
|
||||
return buf, err
|
||||
// Read the attribute key-value pairs. Iterate over pairs rather than
|
||||
// n*2 elements so a count above MaxInt/2 can't overflow int to a
|
||||
// negative loop bound and skip the body.
|
||||
for i := 0; i < n; i++ {
|
||||
for pair := 0; pair < 2; pair++ {
|
||||
buf, err = r.readRawReplyBuf(buf)
|
||||
if err != nil {
|
||||
return buf, err
|
||||
}
|
||||
}
|
||||
}
|
||||
// Read the command reply that follows the attribute
|
||||
@@ -948,11 +1021,13 @@ func (r *Reader) readRawReplyWriteTo(w io.Writer) (int64, error) {
|
||||
}
|
||||
return written, err
|
||||
}
|
||||
for i := 0; i < count*2; i++ {
|
||||
n, err := r.readRawReplyWriteTo(w)
|
||||
written += n
|
||||
if err != nil {
|
||||
return written, err
|
||||
for i := 0; i < count; i++ {
|
||||
for pair := 0; pair < 2; pair++ {
|
||||
n, err := r.readRawReplyWriteTo(w)
|
||||
written += n
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
@@ -967,12 +1042,16 @@ func (r *Reader) readRawReplyWriteTo(w io.Writer) (int64, error) {
|
||||
}
|
||||
return written, err
|
||||
}
|
||||
// Read the attribute key-value pairs
|
||||
for i := 0; i < count*2; i++ {
|
||||
n, err := r.readRawReplyWriteTo(w)
|
||||
written += n
|
||||
if err != nil {
|
||||
return written, err
|
||||
// Read the attribute key-value pairs. Iterate over pairs rather than
|
||||
// count*2 elements so a count above MaxInt/2 can't overflow int to a
|
||||
// negative loop bound and skip the body.
|
||||
for i := 0; i < count; i++ {
|
||||
for pair := 0; pair < 2; pair++ {
|
||||
n, err := r.readRawReplyWriteTo(w)
|
||||
written += n
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
}
|
||||
// Read the command reply that follows the attribute
|
||||
|
||||
+1
-1
@@ -118,7 +118,7 @@ func (w *Writer) WriteArg(v interface{}) error {
|
||||
return w.uint(uint64(v))
|
||||
case *uint8:
|
||||
if v == nil {
|
||||
return w.string("")
|
||||
return w.uint(0)
|
||||
}
|
||||
return w.uint(uint64(*v))
|
||||
case uint16:
|
||||
|
||||
+24
-30
@@ -14,6 +14,28 @@ var semTimers = sync.Pool{
|
||||
},
|
||||
}
|
||||
|
||||
// putSemTimer stops a pooled timer and drains a stale fire before reuse,
|
||||
// portably across both timer-channel semantics (the drain must never block):
|
||||
//
|
||||
// - main module on go >= 1.23 (synchronous channels): Stop returns TRUE for
|
||||
// an expired-but-undelivered fire — the delivery is aborted — and false
|
||||
// only once the value was actually received. With the sole receiver
|
||||
// being Acquire's own select, the drain branch is unreachable; the
|
||||
// select-with-default is a safety net so a future semantics shift cannot
|
||||
// turn it into a blocking receive (reviewed on #3942).
|
||||
// - GODEBUG=asynctimerchan=1 (consumer main module on go < 1.23, old
|
||||
// buffered channels): Stop returns false and the fired value sits in the
|
||||
// buffer; the drain consumes it so the timer is clean for Reset-reuse.
|
||||
func putSemTimer(t *time.Timer) {
|
||||
if !t.Stop() {
|
||||
select {
|
||||
case <-t.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
semTimers.Put(t)
|
||||
}
|
||||
|
||||
// FastSemaphore is a channel-based semaphore optimized for performance.
|
||||
// It uses a fast path that avoids timer allocation when tokens are available.
|
||||
// The channel is pre-filled with tokens: Acquire = receive, Release = send.
|
||||
@@ -70,19 +92,13 @@ func (s *FastSemaphore) Acquire(ctx context.Context, timeout time.Duration, time
|
||||
|
||||
// Slow path: need to wait with timeout
|
||||
timer := semTimers.Get().(*time.Timer)
|
||||
defer semTimers.Put(timer)
|
||||
defer putSemTimer(timer)
|
||||
timer.Reset(timeout)
|
||||
|
||||
select {
|
||||
case <-s.tokens:
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return timeoutErr
|
||||
@@ -152,42 +168,20 @@ func (s *FIFOSemaphore) TryAcquire() bool {
|
||||
func (s *FIFOSemaphore) Acquire(ctx context.Context, timeout time.Duration, timeoutErr error) error {
|
||||
// No fast path - always use timer to guarantee FIFO
|
||||
timer := semTimers.Get().(*time.Timer)
|
||||
defer semTimers.Put(timer)
|
||||
defer putSemTimer(timer)
|
||||
timer.Reset(timeout)
|
||||
|
||||
select {
|
||||
case <-s.tokens:
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return timeoutErr
|
||||
}
|
||||
}
|
||||
|
||||
// AcquireBlocking acquires a token, blocking indefinitely until one is available.
|
||||
func (s *FIFOSemaphore) AcquireBlocking() {
|
||||
<-s.tokens
|
||||
}
|
||||
|
||||
// Release releases a token back to the semaphore.
|
||||
func (s *FIFOSemaphore) Release() {
|
||||
s.tokens <- struct{}{}
|
||||
}
|
||||
|
||||
// Close closes the semaphore, unblocking all waiting goroutines.
|
||||
// After close, all Acquire calls will receive a closed channel signal.
|
||||
func (s *FIFOSemaphore) Close() {
|
||||
close(s.tokens)
|
||||
}
|
||||
|
||||
// Len returns the current number of acquired tokens.
|
||||
func (s *FIFOSemaphore) Len() int32 {
|
||||
return s.max - int32(len(s.tokens))
|
||||
}
|
||||
|
||||
+8
@@ -46,6 +46,14 @@ func (it *ScanIterator) Next(ctx context.Context) bool {
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
// Await the fetch before reading page/cursor: on the deferred
|
||||
// autopipeline face process() only enqueues, and reading the raw
|
||||
// fields of a not-yet-executed command would spin re-issuing SCANs
|
||||
// with a stale cursor forever. Err() blocks until executed there and
|
||||
// is a no-op read everywhere else.
|
||||
if err := it.cmd.Err(); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
it.pos = 1
|
||||
|
||||
|
||||
+21
-6
@@ -96,6 +96,7 @@ func newJSONCmd(ctx context.Context, args ...interface{}) *JSONCmd {
|
||||
}
|
||||
|
||||
func (cmd *JSONCmd) String() string {
|
||||
cmd.await()
|
||||
return cmdString(cmd, cmd.val)
|
||||
}
|
||||
|
||||
@@ -105,6 +106,7 @@ func (cmd *JSONCmd) SetVal(val string) {
|
||||
|
||||
// Val returns the result of the JSON.GET command as a string.
|
||||
func (cmd *JSONCmd) Val() string {
|
||||
cmd.await()
|
||||
if len(cmd.val) == 0 && cmd.expanded != nil {
|
||||
val, err := json.Marshal(cmd.expanded)
|
||||
if err != nil {
|
||||
@@ -119,11 +121,13 @@ func (cmd *JSONCmd) Val() string {
|
||||
}
|
||||
|
||||
func (cmd *JSONCmd) Result() (string, error) {
|
||||
cmd.await()
|
||||
return cmd.Val(), cmd.Err()
|
||||
}
|
||||
|
||||
// Expanded returns the result of the JSON.GET command as unmarshalled JSON.
|
||||
func (cmd *JSONCmd) Expanded() (interface{}, error) {
|
||||
cmd.await()
|
||||
if len(cmd.val) != 0 && cmd.expanded == nil {
|
||||
err := json.Unmarshal([]byte(cmd.val), &cmd.expanded)
|
||||
if err != nil {
|
||||
@@ -136,15 +140,18 @@ func (cmd *JSONCmd) Expanded() (interface{}, error) {
|
||||
|
||||
func (cmd *JSONCmd) readReply(rd *proto.Reader) error {
|
||||
// nil response from JSON.(M)GET (cmd.baseCmd.err will be "redis: nil")
|
||||
// This happens when the key doesn't exist
|
||||
if cmd.baseCmd.Err() == Nil {
|
||||
// This happens when the key doesn't exist.
|
||||
// Use rawErr() (not Err()): readReply runs inside the batch's Exec, before
|
||||
// the autopipeline batch's done channel is closed, so Err()->await() would
|
||||
// deadlock on the very Exec that is calling readReply.
|
||||
if cmd.baseCmd.rawErr() == Nil {
|
||||
cmd.val = ""
|
||||
return Nil
|
||||
}
|
||||
|
||||
// Handle other base command errors
|
||||
if cmd.baseCmd.Err() != nil {
|
||||
return cmd.baseCmd.Err()
|
||||
if cmd.baseCmd.rawErr() != nil {
|
||||
return cmd.baseCmd.rawErr()
|
||||
}
|
||||
|
||||
if readType, err := rd.PeekReplyType(); err != nil {
|
||||
@@ -212,6 +219,7 @@ func NewJSONSliceCmd(ctx context.Context, args ...interface{}) *JSONSliceCmd {
|
||||
}
|
||||
|
||||
func (cmd *JSONSliceCmd) String() string {
|
||||
cmd.await()
|
||||
return cmdString(cmd, cmd.val)
|
||||
}
|
||||
|
||||
@@ -220,15 +228,19 @@ func (cmd *JSONSliceCmd) SetVal(val []interface{}) {
|
||||
}
|
||||
|
||||
func (cmd *JSONSliceCmd) Val() []interface{} {
|
||||
cmd.await()
|
||||
return cmd.val
|
||||
}
|
||||
|
||||
func (cmd *JSONSliceCmd) Result() ([]interface{}, error) {
|
||||
cmd.await()
|
||||
return cmd.val, cmd.err
|
||||
}
|
||||
|
||||
func (cmd *JSONSliceCmd) readReply(rd *proto.Reader) error {
|
||||
if cmd.baseCmd.Err() == Nil {
|
||||
// rawErr(), not Err(): readReply runs inside Exec before the batch's done
|
||||
// channel closes, so Err()->await() would deadlock (see JSONCmd.readReply).
|
||||
if cmd.baseCmd.rawErr() == Nil {
|
||||
cmd.val = nil
|
||||
return Nil
|
||||
}
|
||||
@@ -238,7 +250,7 @@ func (cmd *JSONSliceCmd) readReply(rd *proto.Reader) error {
|
||||
} else if readType == proto.RespArray {
|
||||
response, err := rd.ReadReply()
|
||||
if err != nil {
|
||||
return nil
|
||||
return err
|
||||
} else {
|
||||
cmd.val = response.([]interface{})
|
||||
}
|
||||
@@ -299,6 +311,7 @@ func NewIntPointerSliceCmd(ctx context.Context, args ...interface{}) *IntPointer
|
||||
}
|
||||
|
||||
func (cmd *IntPointerSliceCmd) String() string {
|
||||
cmd.await()
|
||||
return cmdString(cmd, cmd.val)
|
||||
}
|
||||
|
||||
@@ -307,10 +320,12 @@ func (cmd *IntPointerSliceCmd) SetVal(val []*int64) {
|
||||
}
|
||||
|
||||
func (cmd *IntPointerSliceCmd) Val() []*int64 {
|
||||
cmd.await()
|
||||
return cmd.val
|
||||
}
|
||||
|
||||
func (cmd *IntPointerSliceCmd) Result() ([]*int64, error) {
|
||||
cmd.await()
|
||||
return cmd.val, cmd.err
|
||||
}
|
||||
|
||||
|
||||
+65
@@ -34,6 +34,8 @@ type ListCmdable interface {
|
||||
RPushX(ctx context.Context, key string, values ...interface{}) *IntCmd
|
||||
LMove(ctx context.Context, source, destination, srcpos, destpos string) *StringCmd
|
||||
BLMove(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration) *StringCmd
|
||||
LMoveM(ctx context.Context, source, destination, srcpos, destpos string, args LMoveMArgs) *StringSliceCmd
|
||||
BLMoveM(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration, args LMoveMArgs) *StringSliceCmd
|
||||
}
|
||||
|
||||
func (c cmdable) BLPop(ctx context.Context, timeout time.Duration, keys ...string) *StringSliceCmd {
|
||||
@@ -156,6 +158,45 @@ type LPosArgs struct {
|
||||
Rank, MaxLen int64
|
||||
}
|
||||
|
||||
// LMoveMMode is the count semantics for LMOVEM/BLMOVEM.
|
||||
type LMoveMMode string
|
||||
|
||||
const (
|
||||
LMoveMCount LMoveMMode = "COUNT" // up to Count
|
||||
LMoveMExactly LMoveMMode = "EXACTLY" // exactly Count, or nothing
|
||||
)
|
||||
|
||||
// LMoveMOrder is the destination ordering for LMOVEM/BLMOVEM.
|
||||
type LMoveMOrder string
|
||||
|
||||
const (
|
||||
LMoveMOBO LMoveMOrder = "OBO" // one-by-one, order reversed
|
||||
LMoveMBulk LMoveMOrder = "BULK" // preserve order
|
||||
)
|
||||
|
||||
// LMoveMArgs configures the optional count group of LMOVEM/BLMOVEM.
|
||||
// Count <= 0 moves a single element. Mode defaults to COUNT, Order to BULK.
|
||||
type LMoveMArgs struct {
|
||||
Mode LMoveMMode
|
||||
Count int64
|
||||
Order LMoveMOrder
|
||||
}
|
||||
|
||||
func (a LMoveMArgs) appendArgs(args []interface{}) []interface{} {
|
||||
if a.Count <= 0 {
|
||||
return args
|
||||
}
|
||||
mode := a.Mode
|
||||
if mode == "" {
|
||||
mode = LMoveMCount
|
||||
}
|
||||
order := a.Order
|
||||
if order == "" {
|
||||
order = LMoveMBulk
|
||||
}
|
||||
return append(args, string(mode), a.Count, string(order))
|
||||
}
|
||||
|
||||
func (c cmdable) LPos(ctx context.Context, key string, value string, a LPosArgs) *IntCmd {
|
||||
args := []interface{}{"lpos", key, value}
|
||||
if a.Rank != 0 {
|
||||
@@ -295,3 +336,27 @@ func (c cmdable) BLMove(
|
||||
_ = c(ctx, cmd)
|
||||
return cmd
|
||||
}
|
||||
|
||||
// LMoveM atomically moves multiple elements between lists (Redis 8.10+).
|
||||
// srcpos/destpos are "LEFT" or "RIGHT". Returns moved elements, or redis.Nil if none.
|
||||
func (c cmdable) LMoveM(ctx context.Context, source, destination, srcpos, destpos string, a LMoveMArgs) *StringSliceCmd {
|
||||
args := make([]interface{}, 5, 8)
|
||||
args[0], args[1], args[2], args[3], args[4] = "lmovem", source, destination, srcpos, destpos
|
||||
args = a.appendArgs(args)
|
||||
cmd := NewStringSliceCmd(ctx, args...)
|
||||
_ = c(ctx, cmd)
|
||||
return cmd
|
||||
}
|
||||
|
||||
// BLMoveM is the blocking variant of LMoveM (Redis 8.10+); timeout 0 blocks forever.
|
||||
// Returns moved elements, or redis.Nil on timeout.
|
||||
func (c cmdable) BLMoveM(ctx context.Context, source, destination, srcpos, destpos string, timeout time.Duration, a LMoveMArgs) *StringSliceCmd {
|
||||
args := make([]interface{}, 6, 9)
|
||||
args[0], args[1], args[2], args[3], args[4] = "blmovem", source, destination, srcpos, destpos
|
||||
args[5] = formatSec(ctx, timeout)
|
||||
args = a.appendArgs(args)
|
||||
cmd := NewStringSliceCmd(ctx, args...)
|
||||
cmd.setReadTimeout(timeout)
|
||||
_ = c(ctx, cmd)
|
||||
return cmd
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user