mirror of
https://github.com/versity/versitygw.git
synced 2026-01-07 04:06:23 +00:00
Merge pull request #534 from versity/ben/metrics
feat: add metrics module for forwarding gateway metrics
This commit is contained in:
Ben McClelland
GitHub
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
@@ -62,6 +63,9 @@ var (
|
||||
iamCacheDisable bool
|
||||
iamCacheTTL int
|
||||
iamCachePrune int
|
||||
metricsService string
|
||||
statsdServers string
|
||||
dogstatsServers string
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -398,6 +402,27 @@ func initFlags() []cli.Flag {
|
||||
EnvVars: []string{"VGW_READ_ONLY"},
|
||||
Destination: &readonly,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-service-name",
|
||||
Usage: "service name tag for metrics, hostname if blank",
|
||||
EnvVars: []string{"VGW_METRICS_SERVICE_NAME"},
|
||||
Aliases: []string{"msn"},
|
||||
Destination: &metricsService,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-statsd-servers",
|
||||
Usage: "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
|
||||
EnvVars: []string{"VGW_METRICS_STATSD_SERVERS"},
|
||||
Aliases: []string{"mss"},
|
||||
Destination: &statsdServers,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "metrics-dogstatsd-servers",
|
||||
Usage: "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
|
||||
EnvVars: []string{"VGW_METRICS_DOGSTATS_SERVERS"},
|
||||
Aliases: []string{"mds"},
|
||||
Destination: &dogstatsServers,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -508,6 +533,15 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
return fmt.Errorf("setup logger: %w", err)
|
||||
}
|
||||
|
||||
metricsManager, err := metrics.NewManager(ctx, metrics.Config{
|
||||
ServiceName: metricsService,
|
||||
StatsdServers: statsdServers,
|
||||
DogStatsdServers: dogstatsServers,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("init metrics manager: %w", err)
|
||||
}
|
||||
|
||||
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
|
||||
KafkaURL: kafkaURL,
|
||||
KafkaTopic: kafkaTopic,
|
||||
@@ -524,7 +558,7 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
|
||||
Access: rootUserAccess,
|
||||
Secret: rootUserSecret,
|
||||
}, port, region, iam, logger, evSender, opts...)
|
||||
}, port, region, iam, logger, evSender, metricsManager, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init gateway: %v", err)
|
||||
}
|
||||
@@ -587,5 +621,9 @@ Loop:
|
||||
}
|
||||
}
|
||||
|
||||
if metricsManager != nil {
|
||||
metricsManager.Close()
|
||||
}
|
||||
|
||||
return saveErr
|
||||
}
|
||||
|
||||
3
go.mod
3
go.mod
@@ -16,6 +16,7 @@ require (
|
||||
github.com/nats-io/nats.go v1.35.0
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/segmentio/kafka-go v0.4.47
|
||||
github.com/smira/go-statsd v1.3.3
|
||||
github.com/urfave/cli/v2 v2.27.2
|
||||
github.com/valyala/fasthttp v1.54.0
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
|
||||
@@ -26,6 +27,8 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
|
||||
github.com/DataDog/datadog-go/v5 v5.5.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect
|
||||
|
||||
25
go.sum
25
go.sum
@@ -12,6 +12,11 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
|
||||
github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
|
||||
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
@@ -68,6 +73,7 @@ github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84Egg
|
||||
github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@@ -116,6 +122,7 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ
|
||||
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
|
||||
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -127,9 +134,13 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
|
||||
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI=
|
||||
github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
@@ -154,8 +165,10 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
@@ -163,11 +176,14 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf
|
||||
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
|
||||
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
@@ -178,10 +194,16 @@ golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -215,9 +237,12 @@ golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
63
metrics/actions.go
Normal file
63
metrics/actions.go
Normal file
@@ -0,0 +1,63 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
var (
|
||||
ActionUndetected = "ActionUnDetected"
|
||||
ActionAbortMultipartUpload = "s3_AbortMultipartUpload"
|
||||
ActionCompleteMultipartUpload = "s3_CompleteMultipartUpload"
|
||||
ActionCopyObject = "s3_CopyObject"
|
||||
ActionCreateBucket = "s3_CreateBucket"
|
||||
ActionCreateMultipartUpload = "s3_CreateMultipartUpload"
|
||||
ActionDeleteBucket = "s3_DeleteBucket"
|
||||
ActionDeleteBucketPolicy = "s3_DeleteBucketPolicy"
|
||||
ActionDeleteBucketTagging = "s3_DeleteBucketTagging"
|
||||
ActionDeleteObject = "s3_DeleteObject"
|
||||
ActionDeleteObjectTagging = "s3_DeleteObjectTagging"
|
||||
ActionDeleteObjects = "s3_DeleteObjects"
|
||||
ActionGetBucketAcl = "s3_GetBucketAcl"
|
||||
ActionGetBucketPolicy = "s3_GetBucketPolicy"
|
||||
ActionGetBucketTagging = "s3_GetBucketTagging"
|
||||
ActionGetBucketVersioning = "s3_GetBucketVersioning"
|
||||
ActionGetObject = "s3_GetObject"
|
||||
ActionGetObjectAcl = "s3_GetObjectAcl"
|
||||
ActionGetObjectAttributes = "s3_GetObjectAttributes"
|
||||
ActionGetObjectLegalHold = "s3_GetObjectLegalHold"
|
||||
ActionGetObjectLockConfiguration = "s3_GetObjectLockConfiguration"
|
||||
ActionGetObjectRetention = "s3_GetObjectRetention"
|
||||
ActionGetObjectTagging = "s3_GetObjectTagging"
|
||||
ActionHeadBucket = "s3_HeadBucket"
|
||||
ActionHeadObject = "s3_HeadObject"
|
||||
ActionListAllMyBuckets = "s3_ListAllMyBuckets"
|
||||
ActionListMultipartUploads = "s3_ListMultipartUploads"
|
||||
ActionListObjectVersions = "s3_ListObjectVersions"
|
||||
ActionListObjects = "s3_ListObjects"
|
||||
ActionListObjectsV2 = "s3_ListObjectsV2"
|
||||
ActionListParts = "s3_ListParts"
|
||||
ActionPutBucketAcl = "s3_PutBucketAcl"
|
||||
ActionPutBucketPolicy = "s3_PutBucketPolicy"
|
||||
ActionPutBucketTagging = "s3_PutBucketTagging"
|
||||
ActionPutBucketVersioning = "s3_PutBucketVersioning"
|
||||
ActionPutObject = "s3_PutObject"
|
||||
ActionPutObjectAcl = "s3_PutObjectAcl"
|
||||
ActionPutObjectLegalHold = "s3_PutObjectLegalHold"
|
||||
ActionPutObjectLockConfiguration = "s3_PutObjectLockConfiguration"
|
||||
ActionPutObjectRetention = "s3_PutObjectRetention"
|
||||
ActionPutObjectTagging = "s3_PutObjectTagging"
|
||||
ActionRestoreObject = "s3_RestoreObject"
|
||||
ActionSelectObjectContent = "s3_SelectObjectContent"
|
||||
ActionUploadPart = "s3_UploadPart"
|
||||
ActionUploadPartCopy = "s3_UploadPartCopy"
|
||||
)
|
||||
51
metrics/dogstats.go
Normal file
51
metrics/dogstats.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
dogstats "github.com/DataDog/datadog-go/v5/statsd"
|
||||
)
|
||||
|
||||
// vgwDogStatsd metrics type
|
||||
type vgwDogStatsd struct {
|
||||
c *dogstats.Client
|
||||
}
|
||||
|
||||
var (
|
||||
rateSampleAlways = 1.0
|
||||
)
|
||||
|
||||
// newDogStatsd takes a server address and returns a statsd merics
|
||||
func newDogStatsd(server string, service string) (*vgwDogStatsd, error) {
|
||||
c, err := dogstats.New(server,
|
||||
dogstats.WithMaxMessagesPerPayload(1000),
|
||||
dogstats.WithNamespace("versitygw"),
|
||||
dogstats.WithTags([]string{
|
||||
"service:" + service,
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &vgwDogStatsd{c: c}, nil
|
||||
}
|
||||
|
||||
// Close closes statsd connections
|
||||
func (s *vgwDogStatsd) Close() {
|
||||
s.c.Close()
|
||||
}
|
||||
|
||||
func (t Tag) ddString() string {
|
||||
if t.Value == "" {
|
||||
return t.Key
|
||||
}
|
||||
return fmt.Sprintf("%v:%v", t.Key, t.Value)
|
||||
}
|
||||
|
||||
// Add adds value to key
|
||||
func (s *vgwDogStatsd) Add(module, key string, value int64, tags ...Tag) {
|
||||
stags := make([]string, len(tags))
|
||||
for i, t := range tags {
|
||||
stags[i] = t.ddString()
|
||||
}
|
||||
s.c.Count(fmt.Sprintf("%v.%v", module, key), value, stags, rateSampleAlways)
|
||||
}
|
||||
195
metrics/metrics.go
Normal file
195
metrics/metrics.go
Normal file
@@ -0,0 +1,195 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
// max size of data items to buffer before dropping
|
||||
// new incoming data items
|
||||
dataItemCount = 100000
|
||||
)
|
||||
|
||||
// Tag is added metadata for metrics
|
||||
type Tag struct {
|
||||
// Key is tag name
|
||||
Key string
|
||||
// Value is tag data
|
||||
Value string
|
||||
}
|
||||
|
||||
// Manager is a manager of metrics plugins
|
||||
type Manager struct {
|
||||
wg sync.WaitGroup
|
||||
ctx context.Context
|
||||
|
||||
config Config
|
||||
|
||||
publishers []publisher
|
||||
addDataChan chan datapoint
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ServiceName string
|
||||
StatsdServers string
|
||||
DogStatsdServers string
|
||||
}
|
||||
|
||||
// NewManager initializes metrics plugins and returns a new metrics manager
|
||||
func NewManager(ctx context.Context, conf Config) (*Manager, error) {
|
||||
if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if conf.ServiceName == "" {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get hostname: %w", err)
|
||||
}
|
||||
conf.ServiceName = hostname
|
||||
}
|
||||
|
||||
addDataChan := make(chan datapoint, dataItemCount)
|
||||
|
||||
mgr := &Manager{
|
||||
addDataChan: addDataChan,
|
||||
ctx: ctx,
|
||||
config: conf,
|
||||
}
|
||||
|
||||
// setup statsd endpoints
|
||||
if len(conf.StatsdServers) > 0 {
|
||||
statsdServers := strings.Split(conf.StatsdServers, ",")
|
||||
|
||||
for _, server := range statsdServers {
|
||||
statsd, err := newStatsd(server, conf.ServiceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mgr.publishers = append(mgr.publishers, statsd)
|
||||
}
|
||||
}
|
||||
|
||||
// setup dogstatsd endpoints
|
||||
if len(conf.DogStatsdServers) > 0 {
|
||||
dogStatsdServers := strings.Split(conf.DogStatsdServers, ",")
|
||||
|
||||
for _, server := range dogStatsdServers {
|
||||
dogStatsd, err := newDogStatsd(server, conf.ServiceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mgr.publishers = append(mgr.publishers, dogStatsd)
|
||||
}
|
||||
}
|
||||
|
||||
mgr.wg.Add(1)
|
||||
go mgr.addForwarder(addDataChan)
|
||||
|
||||
return mgr, nil
|
||||
}
|
||||
|
||||
func (m *Manager) Send(err error, action string, count int64) {
|
||||
// In case of Authentication failures, url parsing ...
|
||||
if action == "" {
|
||||
action = ActionUndetected
|
||||
}
|
||||
if err != nil {
|
||||
m.increment(action, "failed_count")
|
||||
} else {
|
||||
m.increment(action, "success_count")
|
||||
}
|
||||
|
||||
switch action {
|
||||
case ActionPutObject:
|
||||
m.add(action, "bytes_written", count)
|
||||
m.increment(action, "object_created_count")
|
||||
case ActionCompleteMultipartUpload:
|
||||
m.increment(action, "object_created_count")
|
||||
case ActionUploadPart:
|
||||
m.add(action, "bytes_written", count)
|
||||
case ActionGetObject:
|
||||
m.add(action, "bytes_read", count)
|
||||
case ActionDeleteObject:
|
||||
m.increment(action, "object_removed_count")
|
||||
case ActionDeleteObjects:
|
||||
m.add(action, "object_removed_count", count)
|
||||
}
|
||||
}
|
||||
|
||||
// increment increments the key by one
|
||||
func (m *Manager) increment(module, key string, tags ...Tag) {
|
||||
m.add(module, key, 1, tags...)
|
||||
}
|
||||
|
||||
// add adds value to key
|
||||
func (m *Manager) add(module, key string, value int64, tags ...Tag) {
|
||||
if m.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
d := datapoint{
|
||||
module: module,
|
||||
key: key,
|
||||
value: value,
|
||||
tags: tags,
|
||||
}
|
||||
|
||||
select {
|
||||
case m.addDataChan <- d:
|
||||
default:
|
||||
// channel full, drop the updates
|
||||
}
|
||||
}
|
||||
|
||||
// Close closes metrics channels, waits for data to complete, closes all plugins
|
||||
func (m *Manager) Close() {
|
||||
// drain the datapoint channels
|
||||
close(m.addDataChan)
|
||||
m.wg.Wait()
|
||||
|
||||
// close all publishers
|
||||
for _, p := range m.publishers {
|
||||
p.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// publisher is the interface for interacting with the metrics plugins
|
||||
type publisher interface {
|
||||
Add(module, key string, value int64, tags ...Tag)
|
||||
Close()
|
||||
}
|
||||
|
||||
func (m *Manager) addForwarder(addChan <-chan datapoint) {
|
||||
for data := range addChan {
|
||||
for _, s := range m.publishers {
|
||||
s.Add(data.module, data.key, data.value, data.tags...)
|
||||
}
|
||||
}
|
||||
m.wg.Done()
|
||||
}
|
||||
|
||||
type datapoint struct {
|
||||
module string
|
||||
key string
|
||||
value int64
|
||||
tags []Tag
|
||||
}
|
||||
53
metrics/statsd.go
Normal file
53
metrics/statsd.go
Normal file
@@ -0,0 +1,53 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/smira/go-statsd"
|
||||
)
|
||||
|
||||
// vgwStatsd metrics type
|
||||
type vgwStatsd struct {
|
||||
c *statsd.Client
|
||||
}
|
||||
|
||||
// newStatsd takes a server address and returns a statsd merics
|
||||
// Supply service name to be used as a tag to identify the spcific
|
||||
// gateway instance, this may typically be the gateway hostname
|
||||
func newStatsd(server string, service string) (*vgwStatsd, error) {
|
||||
c := statsd.NewClient(
|
||||
server,
|
||||
statsd.MetricPrefix("versitygw."),
|
||||
statsd.TagStyle(statsd.TagFormatInfluxDB),
|
||||
statsd.DefaultTags(statsd.StringTag("service", service)),
|
||||
)
|
||||
return &vgwStatsd{c: c}, nil
|
||||
}
|
||||
|
||||
// Close closes statsd connections
|
||||
func (s *vgwStatsd) Close() {
|
||||
s.c.Close()
|
||||
}
|
||||
|
||||
// Add adds value to key
|
||||
func (s *vgwStatsd) Add(module, key string, value int64, tags ...Tag) {
|
||||
stags := make([]statsd.Tag, len(tags))
|
||||
for i, t := range tags {
|
||||
stags[i] = statsd.StringTag(t.Key, t.Value)
|
||||
}
|
||||
s.c.Incr(fmt.Sprintf("%v.%v", module, key), value, stags...)
|
||||
}
|
||||
@@ -46,10 +46,10 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
|
||||
|
||||
// Logging middlewares
|
||||
app.Use(logger.New())
|
||||
app.Use(middlewares.DecodeURL(nil))
|
||||
app.Use(middlewares.DecodeURL(nil, nil))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, nil, nil, region, false))
|
||||
app.Use(middlewares.VerifyMD5Body(nil))
|
||||
|
||||
server.router.Init(app, be, iam)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -77,7 +77,7 @@ func TestNew(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := New(tt.args.be, tt.args.iam, nil, nil, false, false)
|
||||
got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("New() = %v, want %v", got, tt.want)
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
@@ -40,7 +41,7 @@ type RootUserConfig struct {
|
||||
Secret string
|
||||
}
|
||||
|
||||
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
@@ -54,16 +55,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
ctx.Locals("startTime", time.Now())
|
||||
authorization := ctx.Get("Authorization")
|
||||
if authorization == "" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
|
||||
}
|
||||
|
||||
authData, err := utils.ParseAuthorization(authorization)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
|
||||
if authData.Algorithm != "AWS4-HMAC-SHA256" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger, mm)
|
||||
}
|
||||
|
||||
if authData.Region != region {
|
||||
@@ -71,40 +72,40 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}, logger)
|
||||
}, logger, mm)
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
// Check X-Amz-Date header
|
||||
date := ctx.Get("X-Amz-Date")
|
||||
if date == "" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger, mm)
|
||||
}
|
||||
|
||||
// Parse the date and check the date validity
|
||||
tdate, err := time.Parse(iso8601Format, date)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger, mm)
|
||||
}
|
||||
|
||||
if date[:8] != authData.Date {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger, mm)
|
||||
}
|
||||
|
||||
// Validate the dates difference
|
||||
err = utils.ValidateDate(tdate)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
@@ -125,7 +126,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
|
||||
// Compare the calculated hash with the hash provided
|
||||
if hashPayload != hexPayload {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger, mm)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,13 +135,13 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
if contentLengthStr != "" {
|
||||
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
|
||||
}
|
||||
}
|
||||
|
||||
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
@@ -164,6 +165,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
|
||||
return a.iam.GetUserAccount(access)
|
||||
}
|
||||
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger, mm *metrics.Manager) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
|
||||
}
|
||||
|
||||
@@ -20,13 +20,14 @@ import (
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
// ProcessChunkedBody initializes the chunked upload stream if the
|
||||
// request appears to be a chunked upload
|
||||
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
|
||||
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
|
||||
if decodedLength == "" {
|
||||
@@ -36,7 +37,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
|
||||
|
||||
authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
@@ -51,7 +52,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
|
||||
return cr
|
||||
})
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
@@ -20,12 +20,13 @@ import (
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
@@ -38,16 +39,16 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
|
||||
|
||||
authData, err := utils.ParsePresignedURIParts(ctx)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
@@ -61,7 +62,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
|
||||
|
||||
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return sendResponse(ctx, err, logger, mm)
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
|
||||
@@ -18,17 +18,18 @@ import (
|
||||
"net/url"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
|
||||
func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
reqURL := ctx.Request().URI().String()
|
||||
decoded, err := url.Parse(reqURL)
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
|
||||
}
|
||||
ctx.Path(decoded.Path)
|
||||
return ctx.Next()
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
@@ -27,8 +28,8 @@ type S3ApiRouter struct {
|
||||
WithAdmSrv bool
|
||||
}
|
||||
|
||||
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool, readonly bool) {
|
||||
s3ApiController := controllers.New(be, iam, logger, evs, debug, readonly)
|
||||
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
|
||||
s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
|
||||
|
||||
if sa.WithAdmSrv {
|
||||
adminController := controllers.NewAdminController(iam, be)
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false, false)
|
||||
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, false, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/metrics"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
@@ -39,7 +40,17 @@ type S3ApiServer struct {
|
||||
health string
|
||||
}
|
||||
|
||||
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
|
||||
func New(
|
||||
app *fiber.App,
|
||||
be backend.Backend,
|
||||
root middlewares.RootUserConfig,
|
||||
port, region string,
|
||||
iam auth.IAMService,
|
||||
l s3log.AuditLogger,
|
||||
evs s3event.S3EventSender,
|
||||
mm *metrics.Manager,
|
||||
opts ...Option,
|
||||
) (*S3ApiServer, error) {
|
||||
server := &S3ApiServer{
|
||||
app: app,
|
||||
backend: be,
|
||||
@@ -61,17 +72,17 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
|
||||
return ctx.SendStatus(http.StatusOK)
|
||||
})
|
||||
}
|
||||
app.Use(middlewares.DecodeURL(l))
|
||||
app.Use(middlewares.DecodeURL(l, mm))
|
||||
app.Use(middlewares.RequestLogger(server.debug))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
|
||||
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
|
||||
app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
|
||||
app.Use(middlewares.VerifyMD5Body(l))
|
||||
app.Use(middlewares.AclParser(be, l, server.readonly))
|
||||
|
||||
server.router.Init(app, be, iam, l, evs, server.debug, server.readonly)
|
||||
server.router.Init(app, be, iam, l, evs, mm, server.debug, server.readonly)
|
||||
|
||||
return server, nil
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
|
||||
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
|
||||
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user