mirror of
https://github.com/versity/versitygw.git
synced 2026-01-28 05:52:03 +00:00
Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
905b283421 | ||
|
6fea34acda | ||
|
|
1c29fbfd81 | ||
|
a3b14d3a05 | ||
|
|
cafb57eb33 | ||
|
|
0760467c3d | ||
|
|
4d168da376 | ||
|
|
cde033811f | ||
|
7a56c7e15e | ||
|
|
e21e514997 | ||
|
660709fe6d | ||
|
|
5931d713f2 | ||
|
|
08e5eb02a0 | ||
|
|
7cba952546 | ||
|
|
5d6c0f8b67 | ||
|
|
be17b3fd33 | ||
|
|
e6440da30a | ||
|
|
443da7f9a4 |
16
.github/workflows/system.yml
vendored
16
.github/workflows/system.yml
vendored
@@ -1,19 +1,19 @@
|
||||
name: system tests
|
||||
on: pull_request
|
||||
#on:
|
||||
# workflow_dispatch:
|
||||
# inputs:
|
||||
# run_workflow:
|
||||
# description: 'Run command-line tests'
|
||||
jobs:
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Install ShellCheck
|
||||
run: sudo apt-get install shellcheck
|
||||
|
||||
- name: Run ShellCheck
|
||||
run: shellcheck -S warning ./tests/*.sh
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
@@ -38,5 +38,5 @@ jobs:
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
|
||||
export VERSITY_EXE=./versitygw
|
||||
mkdir /tmp/gw
|
||||
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/s3_bucket_tests.sh
|
||||
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/posix_tests.sh
|
||||
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.default $HOME/bin/bats ./tests/s3_bucket_tests.sh
|
||||
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.default $HOME/bin/bats ./tests/posix_tests.sh
|
||||
|
||||
@@ -27,6 +27,7 @@ archives:
|
||||
# this name template makes the OS and Arch compatible with the results of uname.
|
||||
name_template: >-
|
||||
{{ .ProjectName }}_
|
||||
v{{ .Version }}_
|
||||
{{- title .Os }}_
|
||||
{{- if eq .Arch "amd64" }}x86_64
|
||||
{{- else if eq .Arch "386" }}i386
|
||||
|
||||
18
go.mod
18
go.mod
@@ -3,9 +3,9 @@ module github.com/versity/versitygw
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
|
||||
github.com/aws/smithy-go v1.19.0
|
||||
@@ -16,13 +16,13 @@ require (
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/segmentio/kafka-go v0.4.47
|
||||
github.com/urfave/cli/v2 v2.27.1
|
||||
github.com/valyala/fasthttp v1.51.0
|
||||
github.com/valyala/fasthttp v1.52.0
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
|
||||
golang.org/x/sys v0.16.0
|
||||
golang.org/x/sys v0.17.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
|
||||
@@ -38,13 +38,13 @@ require (
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.18 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
golang.org/x/crypto v0.18.0 // indirect
|
||||
golang.org/x/net v0.20.0 // indirect
|
||||
golang.org/x/crypto v0.19.0 // indirect
|
||||
golang.org/x/net v0.21.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.16
|
||||
@@ -57,7 +57,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/klauspost/compress v1.17.2 // indirect
|
||||
github.com/klauspost/compress v1.17.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
|
||||
36
go.sum
36
go.sum
@@ -1,20 +1,20 @@
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 h1:IfFdxTUDiV58iZqPKgyWiz4X4fCxZeQ1pTQPImLYXpY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
|
||||
@@ -76,8 +76,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
|
||||
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
@@ -118,8 +118,8 @@ github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
|
||||
github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
|
||||
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
|
||||
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
|
||||
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
|
||||
@@ -137,8 +137,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
||||
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
|
||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@@ -147,8 +147,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
|
||||
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
|
||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -165,8 +165,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
|
||||
@@ -22,6 +22,30 @@ func TestAuthentication(s *S3Conf) {
|
||||
Authentication_signature_error_incorrect_secret_key(s)
|
||||
}
|
||||
|
||||
func TestPresignedAuthentication(s *S3Conf) {
|
||||
PresignedAuth_missing_algo_query_param(s)
|
||||
PresignedAuth_unsupported_algorithm(s)
|
||||
PresignedAuth_missing_credentials_query_param(s)
|
||||
PresignedAuth_malformed_creds_invalid_parts(s)
|
||||
PresignedAuth_malformed_creds_invalid_parts(s)
|
||||
PresignedAuth_creds_incorrect_service(s)
|
||||
PresignedAuth_creds_incorrect_region(s)
|
||||
PresignedAuth_creds_invalid_date(s)
|
||||
PresignedAuth_missing_date_query(s)
|
||||
PresignedAuth_dates_mismatch(s)
|
||||
PresignedAuth_non_existing_access_key_id(s)
|
||||
PresignedAuth_missing_signed_headers_query_param(s)
|
||||
PresignedAuth_missing_expiration_query_param(s)
|
||||
PresignedAuth_invalid_expiration_query_param(s)
|
||||
PresignedAuth_negative_expiration_query_param(s)
|
||||
PresignedAuth_exceeding_expiration_query_param(s)
|
||||
PresignedAuth_expired_request(s)
|
||||
PresignedAuth_incorrect_secret_key(s)
|
||||
PresignedAuth_PutObject_success(s)
|
||||
PresignedAuth_Put_GetObject_with_data(s)
|
||||
PresignedAuth_UploadPart(s)
|
||||
}
|
||||
|
||||
func TestCreateBucket(s *S3Conf) {
|
||||
CreateBucket_invalid_bucket_name(s)
|
||||
CreateBucket_existing_bucket(s)
|
||||
@@ -219,6 +243,7 @@ func TestGetBucketAcl(s *S3Conf) {
|
||||
|
||||
func TestFullFlow(s *S3Conf) {
|
||||
TestAuthentication(s)
|
||||
TestPresignedAuthentication(s)
|
||||
TestCreateBucket(s)
|
||||
TestHeadBucket(s)
|
||||
TestListBuckets(s)
|
||||
@@ -277,6 +302,27 @@ func GetIntTests() IntTests {
|
||||
"Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash,
|
||||
"Authentication_incorrect_md5": Authentication_incorrect_md5,
|
||||
"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
|
||||
"PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param,
|
||||
"PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm,
|
||||
"PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param,
|
||||
"PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts,
|
||||
"PresignedAuth_creds_invalid_terminator": PresignedAuth_creds_invalid_terminator,
|
||||
"PresignedAuth_creds_incorrect_service": PresignedAuth_creds_incorrect_service,
|
||||
"PresignedAuth_creds_incorrect_region": PresignedAuth_creds_incorrect_region,
|
||||
"PresignedAuth_creds_invalid_date": PresignedAuth_creds_invalid_date,
|
||||
"PresignedAuth_missing_date_query": PresignedAuth_missing_date_query,
|
||||
"PresignedAuth_dates_mismatch": PresignedAuth_dates_mismatch,
|
||||
"PresignedAuth_non_existing_access_key_id": PresignedAuth_non_existing_access_key_id,
|
||||
"PresignedAuth_missing_signed_headers_query_param": PresignedAuth_missing_signed_headers_query_param,
|
||||
"PresignedAuth_missing_expiration_query_param": PresignedAuth_missing_expiration_query_param,
|
||||
"PresignedAuth_invalid_expiration_query_param": PresignedAuth_invalid_expiration_query_param,
|
||||
"PresignedAuth_negative_expiration_query_param": PresignedAuth_negative_expiration_query_param,
|
||||
"PresignedAuth_exceeding_expiration_query_param": PresignedAuth_exceeding_expiration_query_param,
|
||||
"PresignedAuth_expired_request": PresignedAuth_expired_request,
|
||||
"PresignedAuth_incorrect_secret_key": PresignedAuth_incorrect_secret_key,
|
||||
"PresignedAuth_PutObject_success": PresignedAuth_PutObject_success,
|
||||
"PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data,
|
||||
"PresignedAuth_UploadPart": PresignedAuth_UploadPart,
|
||||
"CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name,
|
||||
"CreateBucket_existing_bucket": CreateBucket_existing_bucket,
|
||||
"CreateBucket_as_user": CreateBucket_as_user,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -19,7 +20,8 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
shortTimeout = 10 * time.Second
|
||||
shortTimeout = 10 * time.Second
|
||||
iso8601Format = "20060102T150405Z"
|
||||
)
|
||||
|
||||
func Authentication_empty_auth_header(s *S3Conf) error {
|
||||
@@ -624,6 +626,912 @@ func Authentication_signature_error_incorrect_secret_key(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_missing_algo_query_param(s *S3Conf) error {
|
||||
testName := "PresignedAuth_missing_algo_query_param"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Del("X-Amz-Algorithm")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_unsupported_algorithm(s *S3Conf) error {
|
||||
testName := "PresignedAuth_unsupported_algorithm"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
uri := strings.Replace(v4req.URL, "AWS4-HMAC-SHA256", "AWS4-SHA256", 1)
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, uri, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_missing_credentials_query_param(s *S3Conf) error {
|
||||
testName := "PresignedAuth_missing_credentials_query_param"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Del("X-Amz-Credential")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_malformed_creds_invalid_parts(s *S3Conf) error {
|
||||
testName := "PresignedAuth_malformed_creds_invalid_parts"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Set("X-Amz-Credential", "access/hello/world")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_creds_invalid_terminator(s *S3Conf) error {
|
||||
testName := "PresignedAuth_creds_invalid_terminator"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
uri, err := changeAuthCred(v4req.URL, "aws5_request", credTerminator)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, uri, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_creds_incorrect_service(s *S3Conf) error {
|
||||
testName := "PresignedAuth_creds_incorrect_service"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
uri, err := changeAuthCred(v4req.URL, "sns", credService)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, uri, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureIncorrService)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_creds_incorrect_region(s *S3Conf) error {
|
||||
testName := "PresignedAuth_creds_incorrect_region"
|
||||
cfg := *s
|
||||
|
||||
if cfg.awsRegion == "us-east-1" {
|
||||
cfg.awsRegion = "us-west-1"
|
||||
} else {
|
||||
cfg.awsRegion = "us-east-1"
|
||||
}
|
||||
|
||||
return presignedAuthHandler(&cfg, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", cfg.awsRegion),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_creds_invalid_date(s *S3Conf) error {
|
||||
testName := "PresignedAuth_creds_invalid_date"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
uri, err := changeAuthCred(v4req.URL, "32234Z34", credDate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, uri, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_non_existing_access_key_id(s *S3Conf) error {
|
||||
testName := "PresignedAuth_non_existing_access_key_id"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
uri, err := changeAuthCred(v4req.URL, "a_rarely_existing_access_key_id890asd6f807as6ydf870say", credAccess)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, uri, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_missing_date_query(s *S3Conf) error {
|
||||
testName := "PresignedAuth_missing_date_query"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Del("X-Amz-Date")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_dates_mismatch(s *S3Conf) error {
|
||||
testName := "PresignedAuth_dates_mismatch"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
uri, err := changeAuthCred(v4req.URL, "20060102", credDate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, uri, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_missing_signed_headers_query_param(s *S3Conf) error {
|
||||
testName := "PresignedAuth_missing_signed_headers_query_param"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Del("X-Amz-SignedHeaders")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_missing_expiration_query_param(s *S3Conf) error {
|
||||
testName := "PresignedAuth_missing_expiration_query_param"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Del("X-Amz-Expires")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_invalid_expiration_query_param(s *S3Conf) error {
|
||||
testName := "PresignedAuth_invalid_expiration_query_param"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Set("X-Amz-Expires", "invalid_value")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMalformedExpires)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_negative_expiration_query_param(s *S3Conf) error {
|
||||
testName := "PresignedAuth_negative_expiration_query_param"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Set("X-Amz-Expires", "-3")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrNegativeExpires)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_exceeding_expiration_query_param(s *S3Conf) error {
|
||||
testName := "PresignedAuth_exceeding_expiration_query_param"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Set("X-Amz-Expires", "60580000")
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, urlParsed.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMaximumExpires)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_expired_request(s *S3Conf) error {
|
||||
testName := "PresignedAuth_expired_request"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
urlParsed, err := url.Parse(v4req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
expDate := time.Now().AddDate(0, -1, 0).Format(iso8601Format)
|
||||
|
||||
queries := urlParsed.Query()
|
||||
queries.Set("X-Amz-Date", expDate)
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
uri, err := changeAuthCred(urlParsed.String(), expDate[:8], credDate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, uri, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrExpiredPresignRequest)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_incorrect_secret_key(s *S3Conf) error {
|
||||
testName := "PresignedAuth_incorrect_secret_key"
|
||||
cfg := *s
|
||||
cfg.awsSecret += "x"
|
||||
return presignedAuthHandler(&cfg, testName, func(client *s3.PresignClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_PutObject_success(s *S3Conf) error {
|
||||
testName := "PresignedAuth_PutObject_success"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
bucket := getBucketName()
|
||||
err := setup(s, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("my-obj")})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, v4req.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("expected my-obj to be successfully uploaded and get 200 response status, instead got %v", resp.StatusCode)
|
||||
}
|
||||
|
||||
err = teardown(s, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error {
|
||||
testName := "PresignedAuth_Put_GetObject_with_data"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
bucket, obj := getBucketName(), "my-obj"
|
||||
err := setup(s, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data := "Hello world"
|
||||
body := strings.NewReader(data)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Body: body})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, v4req.URL, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header = v4req.SignedHeader
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("expected my-obj to be successfully uploaded and get %v response status, instead got %v", http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4GetReq, err := client.PresignGetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &obj})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err = http.NewRequest(v4GetReq.Method, v4GetReq.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err = httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("expected get object response status to be %v, instead got %v", http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read get object response body %w", err)
|
||||
}
|
||||
|
||||
fmt.Println(resp.Request.Method, resp.ContentLength, string(respBody))
|
||||
|
||||
if string(respBody) != data {
|
||||
return fmt.Errorf("expected get object response body to be %v, instead got %s", data, respBody)
|
||||
}
|
||||
|
||||
err = teardown(s, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PresignedAuth_UploadPart(s *S3Conf) error {
|
||||
testName := "PresignedAuth_UploadPart"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
|
||||
bucket, key, partNumber := getBucketName(), "my-mp", int32(1)
|
||||
|
||||
err := setup(s, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
clt := s3.NewFromConfig(s.Config())
|
||||
mp, err := createMp(clt, bucket, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignUploadPart(ctx, &s3.UploadPartInput{Bucket: &bucket, Key: &key, UploadId: mp.UploadId, PartNumber: &partNumber})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := http.Client{
|
||||
Timeout: shortTimeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("expected response status code to be %v, instead got %v", http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
|
||||
etag := resp.Header.Get("Etag")
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := clt.ListParts(ctx, &s3.ListPartsInput{Bucket: &bucket, Key: &key, UploadId: mp.UploadId})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(out.Parts) != 1 {
|
||||
return fmt.Errorf("expected mp upload parts length to be 1, instead got %v", len(out.Parts))
|
||||
}
|
||||
if *out.Parts[0].ETag != etag {
|
||||
return fmt.Errorf("expected uploaded part etag to be %v, instead got %v", etag, *out.Parts[0].ETag)
|
||||
}
|
||||
if *out.Parts[0].PartNumber != partNumber {
|
||||
return fmt.Errorf("expected uploaded part part-number to be %v, instead got %v", partNumber, *out.Parts[0].PartNumber)
|
||||
}
|
||||
|
||||
err = teardown(s, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func CreateBucket_invalid_bucket_name(s *S3Conf) error {
|
||||
testName := "CreateBucket_invalid_bucket_name"
|
||||
runF(testName)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"io"
|
||||
rnd "math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
@@ -150,6 +151,20 @@ func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient) error) error {
|
||||
runF(testName)
|
||||
clt := s3.NewPresignClient(s3.NewFromConfig(s.Config()))
|
||||
|
||||
err := handler(clt)
|
||||
if err != nil {
|
||||
failF("%v: %v", testName, err)
|
||||
return fmt.Errorf("%v: %w", testName, err)
|
||||
}
|
||||
|
||||
passF(testName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
|
||||
req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
@@ -551,3 +566,26 @@ func genRandString(length int) string {
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
const (
|
||||
credAccess int = iota
|
||||
credDate
|
||||
credRegion
|
||||
credService
|
||||
credTerminator
|
||||
)
|
||||
|
||||
func changeAuthCred(uri, newVal string, index int) (string, error) {
|
||||
urlParsed, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
creds := strings.Split(queries.Get("X-Amz-Credential"), "/")
|
||||
creds[index] = newVal
|
||||
queries.Set("X-Amz-Credential", strings.Join(creds, "/"))
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
return urlParsed.String(), nil
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -44,6 +44,12 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
// If account is set in context locals, it means it was presigned url case
|
||||
_, ok := ctx.Locals("account").(auth.Account)
|
||||
if ok {
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
ctx.Locals("region", region)
|
||||
ctx.Locals("startTime", time.Now())
|
||||
authorization := ctx.Get("Authorization")
|
||||
@@ -96,7 +102,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
}
|
||||
|
||||
// Validate the dates difference
|
||||
err = validateDate(tdate)
|
||||
err = utils.ValidateDate(tdate)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
@@ -158,29 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
|
||||
return a.iam.GetUserAccount(access)
|
||||
}
|
||||
|
||||
func validateDate(date time.Time) error {
|
||||
now := time.Now().UTC()
|
||||
diff := date.Unix() - now.Unix()
|
||||
|
||||
// Checks the dates difference to be less than a minute
|
||||
if diff > 60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
if diff < -60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
69
s3api/middlewares/presign-auth.go
Normal file
69
s3api/middlewares/presign-auth.go
Normal file
@@ -0,0 +1,69 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
if ctx.Query("X-Amz-Signature") == "" {
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
ctx.Locals("region", region)
|
||||
ctx.Locals("startTime", time.Now())
|
||||
|
||||
authData, err := utils.ParsePresignedURIParts(ctx)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug)
|
||||
})
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
|
||||
app.Use(middlewares.RequestLogger(server.debug))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
|
||||
app.Use(middlewares.VerifyMD5Body(l))
|
||||
|
||||
@@ -126,16 +126,19 @@ func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string,
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
|
||||
AccessKeyID: auth.Access,
|
||||
SecretAccessKey: secret,
|
||||
}, req, checksum, service, auth.Region, tdate, func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
signErr := signer.SignHTTP(req.Context(),
|
||||
aws.Credentials{
|
||||
AccessKeyID: auth.Access,
|
||||
SecretAccessKey: secret,
|
||||
},
|
||||
req, checksum, service, auth.Region, tdate,
|
||||
func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("sign generated http request: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/valyala/fasthttp/fasthttputil"
|
||||
)
|
||||
|
||||
func TestAuthParse(t *testing.T) {
|
||||
@@ -10,18 +17,26 @@ func TestAuthParse(t *testing.T) {
|
||||
authstr string // Authorization string
|
||||
algo string
|
||||
sig string
|
||||
}{{
|
||||
name: "restic",
|
||||
authstr: "AWS4-HMAC-SHA256 Credential=user/20240116/us-east-1/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
|
||||
algo: "AWS4-HMAC-SHA256",
|
||||
sig: "d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
|
||||
},
|
||||
}{
|
||||
{
|
||||
name: "restic",
|
||||
authstr: "AWS4-HMAC-SHA256 Credential=user/20240116/us-east-1/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
|
||||
algo: "AWS4-HMAC-SHA256",
|
||||
sig: "d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
|
||||
},
|
||||
{
|
||||
name: "aws eaxample",
|
||||
authstr: "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-date, Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
|
||||
algo: "AWS4-HMAC-SHA256",
|
||||
sig: "fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
|
||||
}}
|
||||
},
|
||||
{
|
||||
name: "s3browser",
|
||||
authstr: "AWS4-HMAC-SHA256 Credential=access_key/20240206/us-east-1/s3/aws4_request,SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, Signature=37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6",
|
||||
algo: "AWS4-HMAC-SHA256",
|
||||
sig: "37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6",
|
||||
},
|
||||
}
|
||||
|
||||
for _, v := range vectors {
|
||||
t.Run(v.name, func(t *testing.T) {
|
||||
@@ -38,3 +53,78 @@ func TestAuthParse(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 2024/02/06 21:03:28 Request headers:
|
||||
// 2024/02/06 21:03:28 Host: 172.21.0.160:11000
|
||||
// 2024/02/06 21:03:28 User-Agent: S3 Browser/11.5.7 (https://s3browser.com)
|
||||
// 2024/02/06 21:03:28 Authorization: AWS4-HMAC-SHA256 Credential=access_key/20240206/us-east-1/s3/aws4_request,SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, Signature=37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6
|
||||
// 2024/02/06 21:03:28 X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
// 2024/02/06 21:03:28 X-Amz-Date: 20240206T210328Z
|
||||
func Test_Client_UserAgent(t *testing.T) {
|
||||
signedHdrs := []string{"host", "user-agent", "x-amz-content-sha256", "x-amz-date"}
|
||||
access := "access_key"
|
||||
secret := "secret_key"
|
||||
region := "us-east-1"
|
||||
host := "172.21.0.160:11000"
|
||||
agent := "S3 Browser/11.5.7 (https://s3browser.com)"
|
||||
expectedSig := "37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6"
|
||||
dateStr := "20240206T210328Z"
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
|
||||
tdate, err := time.Parse(iso8601Format, dateStr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
app.Get("/", func(c *fiber.Ctx) error {
|
||||
req, err := createHttpRequestFromCtx(c, signedHdrs, int64(c.Request().Header.ContentLength()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req.Host = host
|
||||
req.Header.Add("X-Amz-Content-Sha256", zeroLenSig)
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(),
|
||||
aws.Credentials{
|
||||
AccessKeyID: access,
|
||||
SecretAccessKey: secret,
|
||||
},
|
||||
req, zeroLenSig, service, region, tdate,
|
||||
func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
})
|
||||
if signErr != nil {
|
||||
t.Fatalf("sign generated http request: %v", err)
|
||||
}
|
||||
|
||||
genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if genAuth.Signature != expectedSig {
|
||||
t.Errorf("SIG: %v\nexpected: %v\n", genAuth.Signature, expectedSig)
|
||||
}
|
||||
|
||||
return c.Send(c.Request().Header.UserAgent())
|
||||
})
|
||||
|
||||
ln := fasthttputil.NewInmemoryListener()
|
||||
go func() {
|
||||
err := app.Listener(ln)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
c := fiber.AcquireClient()
|
||||
c.UserAgent = agent
|
||||
a := c.Get("http://example.com")
|
||||
a.HostClient.Dial = func(_ string) (net.Conn, error) { return ln.Dial() }
|
||||
a.String()
|
||||
fiber.ReleaseClient(c)
|
||||
}
|
||||
|
||||
243
s3api/utils/presign-auth-reader.go
Normal file
243
s3api/utils/presign-auth-reader.go
Normal file
@@ -0,0 +1,243 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/smithy-go/logging"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
const (
|
||||
unsignedPayload string = "UNSIGNED-PAYLOAD"
|
||||
)
|
||||
|
||||
// PresignedAuthReader is an io.Reader that validates presigned request authorization
|
||||
// once the underlying reader returns io.EOF. This is needed for streaming
|
||||
// data requests where the data size is not known until
|
||||
// the data is completely read.
|
||||
type PresignedAuthReader struct {
|
||||
ctx *fiber.Ctx
|
||||
auth AuthData
|
||||
secret string
|
||||
r io.Reader
|
||||
debug bool
|
||||
}
|
||||
|
||||
func NewPresignedAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *PresignedAuthReader {
|
||||
return &PresignedAuthReader{
|
||||
ctx: ctx,
|
||||
r: r,
|
||||
auth: auth,
|
||||
secret: secret,
|
||||
debug: debug,
|
||||
}
|
||||
}
|
||||
|
||||
// Read allows *PresignedAuthReader to be used as an io.Reader
|
||||
func (pr *PresignedAuthReader) Read(p []byte) (int, error) {
|
||||
n, err := pr.r.Read(p)
|
||||
|
||||
if errors.Is(err, io.EOF) {
|
||||
cerr := CheckPresignedSignature(pr.ctx, pr.auth, pr.secret, pr.debug)
|
||||
if cerr != nil {
|
||||
return n, cerr
|
||||
}
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// CheckPresignedSignature validates presigned request signature
|
||||
func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, debug bool) error {
|
||||
signedHdrs := strings.Split(auth.SignedHeaders, ";")
|
||||
|
||||
var contentLength int64
|
||||
var err error
|
||||
contentLengthStr := ctx.Get("Content-Length")
|
||||
if contentLengthStr != "" {
|
||||
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new http request instance from fasthttp request
|
||||
req, err := createPresignedHttpRequestFromCtx(ctx, signedHdrs, contentLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create http request from context: %w", err)
|
||||
}
|
||||
|
||||
date, _ := time.Parse(iso8601Format, auth.Date)
|
||||
|
||||
signer := v4.NewSigner()
|
||||
uri, _, signErr := signer.PresignHTTP(ctx.Context(), aws.Credentials{
|
||||
AccessKeyID: auth.Access,
|
||||
SecretAccessKey: secret,
|
||||
}, req, unsignedPayload, service, auth.Region, date, func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("presign generated http request: %w", err)
|
||||
}
|
||||
|
||||
urlParts, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse presigned url: %w", err)
|
||||
}
|
||||
|
||||
signature := urlParts.Query().Get("X-Amz-Signature")
|
||||
if signature != auth.Signature {
|
||||
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
|
||||
//
|
||||
// # ParsePresignedURIParts parses and validates request URL query parameters
|
||||
//
|
||||
// ?X-Amz-Algorithm=AWS4-HMAC-SHA256
|
||||
// &X-Amz-Credential=access-key-id/20130721/us-east-1/s3/aws4_request
|
||||
// &X-Amz-Date=20130721T201207Z
|
||||
// &X-Amz-Expires=86400
|
||||
// &X-Amz-SignedHeaders=host
|
||||
// &X-Amz-Signature=1e68ad45c1db540284a4a1eca3884c293ba1a0ff63ab9db9a15b5b29dfa02cd8
|
||||
func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
|
||||
a := AuthData{}
|
||||
|
||||
// Get and verify algorithm query parameter
|
||||
algo := ctx.Query("X-Amz-Algorithm")
|
||||
if algo == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
if algo != "AWS4-HMAC-SHA256" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)
|
||||
}
|
||||
|
||||
// Parse and validate credentials query parameter
|
||||
credsQuery := ctx.Query("X-Amz-Credential")
|
||||
if credsQuery == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
creds := strings.Split(credsQuery, "/")
|
||||
if len(creds) != 5 {
|
||||
return a, s3err.GetAPIError(s3err.ErrCredMalformed)
|
||||
}
|
||||
if creds[3] != "s3" {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
|
||||
}
|
||||
if creds[4] != "aws4_request" {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
|
||||
}
|
||||
_, err := time.Parse(yyyymmdd, creds[1])
|
||||
if err != nil {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
|
||||
}
|
||||
|
||||
// Parse and validate Date query param
|
||||
date := ctx.Query("X-Amz-Date")
|
||||
if date == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
tdate, err := time.Parse(iso8601Format, date)
|
||||
if err != nil {
|
||||
return a, s3err.GetAPIError(s3err.ErrMalformedDate)
|
||||
}
|
||||
|
||||
if date[:8] != creds[1] {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
|
||||
}
|
||||
|
||||
if ctx.Locals("region") != creds[2] {
|
||||
return a, s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
|
||||
signature := ctx.Query("X-Amz-Signature")
|
||||
if signature == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
signedHdrs := ctx.Query("X-Amz-SignedHeaders")
|
||||
if signedHdrs == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
// Validate X-Amz-Expires query param and check if request is expired
|
||||
err = validateExpiration(ctx.Query("X-Amz-Expires"), tdate)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
|
||||
a.Signature = signature
|
||||
a.Access = creds[0]
|
||||
a.Algorithm = algo
|
||||
a.Region = creds[2]
|
||||
a.SignedHeaders = signedHdrs
|
||||
a.Date = date
|
||||
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func validateExpiration(str string, date time.Time) error {
|
||||
if str == "" {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
exp, err := strconv.Atoi(str)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedExpires)
|
||||
}
|
||||
|
||||
if exp < 0 {
|
||||
return s3err.GetAPIError(s3err.ErrNegativeExpires)
|
||||
}
|
||||
|
||||
if exp > 604800 {
|
||||
return s3err.GetAPIError(s3err.ErrMaximumExpires)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
passed := int(now.Sub(date).Seconds())
|
||||
|
||||
if passed > exp {
|
||||
return s3err.GetAPIError(s3err.ErrExpiredPresignRequest)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
100
s3api/utils/presign-auth-reader_test.go
Normal file
100
s3api/utils/presign-auth-reader_test.go
Normal file
@@ -0,0 +1,100 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
func Test_validateExpiration(t *testing.T) {
|
||||
type args struct {
|
||||
str string
|
||||
date time.Time
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "empty-expiration",
|
||||
args: args{
|
||||
str: "",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrInvalidQueryParams),
|
||||
},
|
||||
{
|
||||
name: "invalid-expiration",
|
||||
args: args{
|
||||
str: "invalid_expiration",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrMalformedExpires),
|
||||
},
|
||||
{
|
||||
name: "negative-expiration",
|
||||
args: args{
|
||||
str: "-320",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrNegativeExpires),
|
||||
},
|
||||
{
|
||||
name: "exceeding-expiration",
|
||||
args: args{
|
||||
str: "6048000",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrMaximumExpires),
|
||||
},
|
||||
{
|
||||
name: "expired value",
|
||||
args: args{
|
||||
str: "200",
|
||||
date: time.Now().AddDate(0, 0, -1),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrExpiredPresignRequest),
|
||||
},
|
||||
{
|
||||
name: "valid expiration",
|
||||
args: args{
|
||||
str: "300",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: nil,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateExpiration(tt.args.str, tt.args.date)
|
||||
// Check for nil case
|
||||
if tt.err == nil && err != nil {
|
||||
t.Errorf("Expected nil error, got: %v", err)
|
||||
return
|
||||
} else if tt.err == nil && err == nil {
|
||||
// Both are nil, no need for further comparison
|
||||
return
|
||||
}
|
||||
|
||||
if err.Error() != tt.err.Error() {
|
||||
t.Errorf("Expected error: %v, got: %v", tt.err, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
48
s3api/utils/sign_hack.go
Normal file
48
s3api/utils/sign_hack.go
Normal file
@@ -0,0 +1,48 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// This is a hack to replace the default IgnoredHeaders in the aws-sdk-go-v2
|
||||
// internal/v4 package. Some AWS applications
|
||||
// (e.g. AWS Java SDK v1, Athena JDBC driver, s3 browser) sign the requests
|
||||
// including the User-Agent header. The aws sdk doesn't allow directly
|
||||
// modifying the ignored header list. Below is a hack to replace this list
|
||||
// with our own.
|
||||
|
||||
type Rule interface {
|
||||
IsValid(value string) bool
|
||||
}
|
||||
type Rules []Rule
|
||||
|
||||
//go:linkname __ignoredHeaders github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4.IgnoredHeaders
|
||||
var __ignoredHeaders unsafe.Pointer
|
||||
|
||||
func init() {
|
||||
// Avoids "go.info.github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4.IgnoredHeaders:
|
||||
// relocation target go.info.github.com/xxx/xxx/xxx.Rules not defined"
|
||||
var ignoredHeaders = (*Rules)(unsafe.Pointer(&__ignoredHeaders))
|
||||
|
||||
// clear the map, and set just the ignored headers we want
|
||||
reflect.ValueOf((*ignoredHeaders)[0]).FieldByName("Rule").Elem().Clear()
|
||||
reflect.ValueOf((*ignoredHeaders)[0]).FieldByName("Rule").Elem().SetMapIndex(
|
||||
reflect.ValueOf("Authorization"), reflect.ValueOf(struct{}{}))
|
||||
reflect.ValueOf((*ignoredHeaders)[0]).FieldByName("Rule").Elem().SetMapIndex(
|
||||
reflect.ValueOf("Expect"), reflect.ValueOf(struct{}{}))
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/valyala/fasthttp"
|
||||
@@ -86,6 +87,66 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
|
||||
return httpReq, nil
|
||||
}
|
||||
|
||||
var (
|
||||
signedQueryArgs = map[string]bool{
|
||||
"X-Amz-Algorithm": true,
|
||||
"X-Amz-Credential": true,
|
||||
"X-Amz-Date": true,
|
||||
"X-Amz-SignedHeaders": true,
|
||||
"X-Amz-Signature": true,
|
||||
}
|
||||
)
|
||||
|
||||
func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
|
||||
req := ctx.Request()
|
||||
var body io.Reader
|
||||
if IsBigDataAction(ctx) {
|
||||
body = req.BodyStream()
|
||||
} else {
|
||||
body = bytes.NewReader(req.Body())
|
||||
}
|
||||
|
||||
uri := string(ctx.Request().URI().Path())
|
||||
isFirst := true
|
||||
|
||||
ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
|
||||
_, ok := signedQueryArgs[string(key)]
|
||||
if !ok {
|
||||
if isFirst {
|
||||
uri += fmt.Sprintf("?%s=%s", key, value)
|
||||
isFirst = false
|
||||
} else {
|
||||
uri += fmt.Sprintf("&%s=%s", key, value)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
|
||||
if err != nil {
|
||||
return nil, errors.New("error in creating an http request")
|
||||
}
|
||||
// Set the request headers
|
||||
req.Header.VisitAll(func(key, value []byte) {
|
||||
keyStr := string(key)
|
||||
if includeHeader(keyStr, signedHdrs) {
|
||||
httpReq.Header.Add(keyStr, string(value))
|
||||
}
|
||||
})
|
||||
|
||||
// Check if Content-Length in signed headers
|
||||
// If content length is non 0, then the header will be included
|
||||
if !includeHeader("Content-Length", signedHdrs) {
|
||||
httpReq.ContentLength = 0
|
||||
} else {
|
||||
httpReq.ContentLength = contentLength
|
||||
}
|
||||
|
||||
// Set the Host header
|
||||
httpReq.Host = string(req.Header.Host())
|
||||
|
||||
return httpReq, nil
|
||||
}
|
||||
|
||||
func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
|
||||
ctx.Response().Header.DisableNormalizing()
|
||||
for key, val := range meta {
|
||||
@@ -149,3 +210,26 @@ func IsBigDataAction(ctx *fiber.Ctx) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ValidateDate(date time.Time) error {
|
||||
now := time.Now().UTC()
|
||||
diff := date.Unix() - now.Unix()
|
||||
|
||||
// Checks the dates difference to be less than a minute
|
||||
if diff > 60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
if diff < -60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
https://doc.s3.amazonaws.com/2006-03-01/AmazonS3.xsd
|
||||
|
||||
see https://blog.aqwari.net/xml-schema-go/
|
||||
|
||||
go install aqwari.net/xml/cmd/xsdgen@latest
|
||||
xsdgen -o s3api_xsd_generated.go -pkg s3response AmazonS3.xsd
|
||||
File diff suppressed because it is too large
Load Diff
@@ -16,6 +16,7 @@ package s3response
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
@@ -107,7 +108,8 @@ type TagSet struct {
|
||||
}
|
||||
|
||||
type Tagging struct {
|
||||
TagSet TagSet `xml:"TagSet"`
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging" json:"-"`
|
||||
TagSet TagSet `xml:"TagSet"`
|
||||
}
|
||||
|
||||
type DeleteObjects struct {
|
||||
@@ -139,3 +141,58 @@ type Bucket struct {
|
||||
Name string `json:"name"`
|
||||
Owner string `json:"owner"`
|
||||
}
|
||||
|
||||
type ListAllMyBucketsResult struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
|
||||
Owner CanonicalUser
|
||||
Buckets ListAllMyBucketsList
|
||||
}
|
||||
|
||||
type ListAllMyBucketsEntry struct {
|
||||
Name string
|
||||
CreationDate time.Time
|
||||
}
|
||||
|
||||
type ListAllMyBucketsList struct {
|
||||
Bucket []ListAllMyBucketsEntry
|
||||
}
|
||||
|
||||
type CanonicalUser struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
type CopyObjectResult struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||
LastModified time.Time
|
||||
ETag string
|
||||
}
|
||||
|
||||
type AccessControlPolicy struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
|
||||
Owner CanonicalUser
|
||||
AccessControlList AccessControlList
|
||||
}
|
||||
|
||||
type AccessControlList struct {
|
||||
Grant []Grant
|
||||
}
|
||||
|
||||
type Grant struct {
|
||||
Grantee Grantee
|
||||
Permission string
|
||||
}
|
||||
|
||||
// Set the following to encode correctly:
|
||||
//
|
||||
// Grantee: s3response.Grantee{
|
||||
// Xsi: "http://www.w3.org/2001/XMLSchema-instance",
|
||||
// Type: "CanonicalUser",
|
||||
// },
|
||||
type Grantee struct {
|
||||
XMLName xml.Name `xml:"Grantee"`
|
||||
Xsi string `xml:"xmlns:xsi,attr,omitempty"`
|
||||
Type string `xml:"xsi:type,attr,omitempty"`
|
||||
ID string
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
AWS_REGION=us-west-2
|
||||
AWS_REGION=us-east-1
|
||||
AWS_PROFILE=versity
|
||||
VERSITY_EXE=./versitygw
|
||||
BACKEND=posix
|
||||
LOCAL_FOLDER=/tmp/gw
|
||||
AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
BUCKET_ONE_NAME=versity-gwtest-bucket-one
|
||||
BUCKET_TWO_NAME=versity-gwtest-bucket-two
|
||||
RECREATE_BUCKETS=true
|
||||
@@ -3,4 +3,7 @@ AWS_PROFILE=versity
|
||||
VERSITY_EXE=./versitygw
|
||||
BACKEND=posix
|
||||
LOCAL_FOLDER=/tmp/gw
|
||||
AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
BUCKET_ONE_NAME=versity-gwtest-bucket-one
|
||||
BUCKET_TWO_NAME=versity-gwtest-bucket-two
|
||||
RECREATE_BUCKETS=true
|
||||
@@ -7,88 +7,82 @@ source ./tests/util_posix.sh
|
||||
# test that changes to local folders and files are reflected on S3
|
||||
@test "test_local_creation_deletion" {
|
||||
|
||||
local bucket_name="versity-gwtest-put-object-test"
|
||||
if [[ $RECREATE_BUCKETS != "true" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
local object_name="test-object"
|
||||
|
||||
bucket_exists_remote_and_local $bucket_name || local bucket_exists=$?
|
||||
if [[ $bucket_exists -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
local object="$bucket_name"/"$object_name"
|
||||
if [[ $bucket_exists -eq 0 ]]; then
|
||||
object_exists_remote_and_local "$object" || local object_exists=$?
|
||||
if [[ $object_exists -eq 2 ]]; then
|
||||
fail "Object existence check error"
|
||||
fi
|
||||
if [[ $object_exists -eq 0 ]]; then
|
||||
delete_object "$object" || local delete_object=$?
|
||||
[[ $delete_object -eq 0 ]] || fail "Failed to delete object"
|
||||
fi
|
||||
delete_bucket $bucket_name || local delete_bucket=$?
|
||||
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
|
||||
fi
|
||||
mkdir "$LOCAL_FOLDER"/$bucket_name
|
||||
touch "$LOCAL_FOLDER"/$object
|
||||
bucket_exists_remote_and_local $bucket_name || local bucket_exists_two=$?
|
||||
mkdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
local object="$BUCKET_ONE_NAME"/"$object_name"
|
||||
touch "$LOCAL_FOLDER"/"$object"
|
||||
|
||||
bucket_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_exists_two=$?
|
||||
[[ $bucket_exists_two -eq 0 ]] || fail "Failed bucket existence check"
|
||||
object_exists_remote_and_local $object || local object_exists_two=$?
|
||||
object_exists_remote_and_local "$object" || local object_exists_two=$?
|
||||
[[ $object_exists_two -eq 0 ]] || fail "Failed object existence check"
|
||||
rm "$LOCAL_FOLDER"/$object
|
||||
|
||||
rm "$LOCAL_FOLDER"/"$object"
|
||||
sleep 1
|
||||
object_not_exists_remote_and_local $object || local object_deleted=$?
|
||||
object_not_exists_remote_and_local "$object" || local object_deleted=$?
|
||||
[[ $object_deleted -eq 0 ]] || fail "Failed object deletion check"
|
||||
rmdir "$LOCAL_FOLDER"/$bucket_name
|
||||
|
||||
rmdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
sleep 1
|
||||
bucket_not_exists_remote_and_local $bucket_name || local bucket_deleted=$?
|
||||
bucket_not_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_deleted=$?
|
||||
[[ $bucket_deleted -eq 0 ]] || fail "Failed bucket deletion check"
|
||||
}
|
||||
|
||||
# test head-object command
|
||||
@test "test_head_object" {
|
||||
|
||||
local bucket_name="versity-gwtest-head-object"
|
||||
local bucket_name=$BUCKET_ONE_NAME
|
||||
local object_name="object-one"
|
||||
|
||||
touch "$object_name"
|
||||
|
||||
if [ -e "$LOCAL_FOLDER"/$bucket_name/$object_name ]; then
|
||||
chmod 755 "$LOCAL_FOLDER"/$bucket_name/$object_name
|
||||
create_test_files $object_name
|
||||
if [ -e "$LOCAL_FOLDER"/"$bucket_name"/$object_name ]; then
|
||||
chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
|
||||
fi
|
||||
check_and_create_bucket $bucket_name || local created=$?
|
||||
setup_bucket "$bucket_name" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating bucket"
|
||||
put_object "$object_name" "$bucket_name"/"$object_name" || local result="$?"
|
||||
put_object "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name" || local result="$?"
|
||||
[[ result -eq 0 ]] || fail "Error adding object one"
|
||||
chmod 000 "$LOCAL_FOLDER"/$bucket_name/$object_name
|
||||
|
||||
chmod 000 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
|
||||
sleep 1
|
||||
object_is_accessible $bucket_name $object_name || local accessible=$?
|
||||
object_is_accessible "$bucket_name" $object_name || local accessible=$?
|
||||
[[ $accessible -eq 1 ]] || fail "Object should be inaccessible"
|
||||
chmod 755 "$LOCAL_FOLDER"/$bucket_name/$object_name
|
||||
|
||||
chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
|
||||
sleep 1
|
||||
object_is_accessible $bucket_name $object_name || local accessible_two=$?
|
||||
object_is_accessible "$bucket_name" $object_name || local accessible_two=$?
|
||||
[[ $accessible_two -eq 0 ]] || fail "Object should be accessible"
|
||||
delete_object $bucket_name/$object_name
|
||||
delete_bucket $bucket_name
|
||||
|
||||
delete_object "$bucket_name"/$object_name
|
||||
delete_bucket_or_contents "$bucket_name"
|
||||
delete_test_files $object_name
|
||||
}
|
||||
|
||||
# check info, accessiblity of bucket
|
||||
@test "test_get_bucket_info" {
|
||||
|
||||
local bucket_name="versity-gwtest-get-bucket-info"
|
||||
|
||||
if [ -e "$LOCAL_FOLDER"/$bucket_name ]; then
|
||||
chmod 755 "$LOCAL_FOLDER"/$bucket_name
|
||||
if [ -e "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME" ]; then
|
||||
chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
sleep 1
|
||||
else
|
||||
create_bucket $bucket_name || local created=$?
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating bucket"
|
||||
fi
|
||||
chmod 000 "$LOCAL_FOLDER"/$bucket_name
|
||||
|
||||
chmod 000 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
sleep 1
|
||||
bucket_is_accessible $bucket_name || local accessible=$?
|
||||
bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible=$?
|
||||
[[ $accessible -eq 1 ]] || fail "Bucket should be inaccessible"
|
||||
chmod 755 "$LOCAL_FOLDER"/$bucket_name
|
||||
|
||||
chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
sleep 1
|
||||
bucket_is_accessible $bucket_name || local accessible_two=$?
|
||||
bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible_two=$?
|
||||
[[ $accessible_two -eq 0 ]] || fail "Bucket should be accessible"
|
||||
delete_bucket $bucket_name
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
}
|
||||
|
||||
@@ -4,122 +4,92 @@ source ./tests/setup.sh
|
||||
source ./tests/util.sh
|
||||
|
||||
# test creation and deletion of bucket on versitygw
|
||||
@test "create_delete_bucket_test" {
|
||||
@test "test_create_delete_bucket" {
|
||||
|
||||
local bucket_name="versity-gwtest-create-delete-bucket-test"
|
||||
if [[ $RECREATE_BUCKETS != "true" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
bucket_exists $bucket_name || local exists=$?
|
||||
if [[ $exists -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
if [[ $exists -eq 0 ]]; then
|
||||
delete_bucket $bucket_name || local delete_result=$?
|
||||
[[ $delete_result -eq 0 ]] || fail "Failed to delete bucket"
|
||||
bucket_exists $bucket_name || local exists_two=$?
|
||||
[[ $exists_two -eq 1 ]] || fail "Failed bucket deletion"
|
||||
fi
|
||||
create_bucket $bucket_name || local create_result=$?
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local create_result=$?
|
||||
[[ $create_result -eq 0 ]] || fail "Failed to create bucket"
|
||||
bucket_exists $bucket_name || local exists_three=$?
|
||||
|
||||
bucket_exists "$BUCKET_ONE_NAME" || local exists_three=$?
|
||||
[[ $exists_three -eq 0 ]] || fail "Failed bucket existence check"
|
||||
delete_bucket $bucket_name || local delete_result_two=$?
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME" || local delete_result_two=$?
|
||||
[[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
|
||||
}
|
||||
|
||||
# test adding and removing an object on versitygw
|
||||
@test "put_object_test" {
|
||||
@test "test_put_object" {
|
||||
|
||||
local bucket_name="versity-gwtest-put-object-test"
|
||||
local object_name="test-object"
|
||||
|
||||
bucket_exists $bucket_name || local bucket_exists=$?
|
||||
if [[ $bucket_exists -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
local object="$bucket_name"/"$object_name"
|
||||
if [[ $bucket_exists -eq 0 ]]; then
|
||||
object_exists "$object" || local object_exists=$?
|
||||
if [[ $object_exists -eq 2 ]]; then
|
||||
fail "Object existence check error"
|
||||
fi
|
||||
if [[ $object_exists -eq 0 ]]; then
|
||||
delete_object "$object" || local delete_object=$?
|
||||
[[ $delete_object -eq 0 ]] || fail "Failed to delete object"
|
||||
fi
|
||||
delete_bucket $bucket_name || local delete_bucket=$?
|
||||
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
|
||||
fi
|
||||
touch "$object_name"
|
||||
create_bucket $bucket_name || local create_bucket=$?
|
||||
[[ $create_bucket -eq 0 ]] || fail "Failed to create bucket"
|
||||
put_object "$object_name" "$object" || local put_object=$?
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local setup_result=$?
|
||||
[[ $setup_result -eq 0 ]] || fail "error setting up bucket"
|
||||
|
||||
create_test_files "$object_name" || local create_result=$?
|
||||
|
||||
object="$BUCKET_ONE_NAME"/$object_name
|
||||
put_object "$test_file_folder"/"$object_name" "$object" || local put_object=$?
|
||||
[[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
|
||||
object_exists "$object" || local object_exists_two=$?
|
||||
[[ $object_exists_two -eq 0 ]] || fail "Object not added to bucket"
|
||||
delete_object "$object" || local delete_object_two=$?
|
||||
[[ $delete_object_two -eq 0 ]] || fail "Failed to delete object"
|
||||
delete_bucket $bucket_name || local delete_bucket=$?
|
||||
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
|
||||
rm "$object_name"
|
||||
object_exists "$object" || local exists_result_one=$?
|
||||
[[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
|
||||
|
||||
delete_object "$object" || local delete_result=$?
|
||||
[[ $delete_result -eq 0 ]] || fail "Failed to delete object"
|
||||
object_exists "$object" || local exists_result_two=$?
|
||||
[[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$object_name"
|
||||
}
|
||||
|
||||
# test listing buckets on versitygw
|
||||
@test "test_list_buckets" {
|
||||
|
||||
bucket_name_one="versity-gwtest-list-one"
|
||||
bucket_name_two="versity-gwtest-list-two"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local setup_result_one=$?
|
||||
[[ $setup_result_one -eq 0 ]] || fail "Bucket one setup error"
|
||||
setup_bucket "$BUCKET_TWO_NAME" || local setup_result_two=$?
|
||||
[[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error"
|
||||
|
||||
bucket_exists $bucket_name_one || local exists=$?
|
||||
if [[ $exists -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
if [[ $exists -eq 1 ]]; then
|
||||
create_bucket $bucket_name_one || local bucket_create_one=$?
|
||||
[[ $bucket_create_one -eq 0 ]] || fail "Failed to create bucket"
|
||||
fi
|
||||
bucket_exists $bucket_name_two || local exists_two=$?
|
||||
if [[ $exists_two -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
if [[ $exists_two -eq 1 ]]; then
|
||||
create_bucket $bucket_name_two || local bucket_create_two=$?
|
||||
[[ $bucket_create_two -eq 0 ]] || fail "Failed to create bucket"
|
||||
fi
|
||||
list_buckets
|
||||
local bucket_one_found=false
|
||||
local bucket_two_found=false
|
||||
for bucket in "${bucket_array[@]}"; do
|
||||
if [ "$bucket" == $bucket_name_one ]; then
|
||||
if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
|
||||
bucket_one_found=true
|
||||
elif [ "$bucket" == $bucket_name_two ]; then
|
||||
elif [ "$bucket" == "$BUCKET_TWO_NAME" ]; then
|
||||
bucket_two_found=true
|
||||
fi
|
||||
if [ $bucket_one_found == true ] && [ $bucket_two_found == true ]; then
|
||||
return
|
||||
break
|
||||
fi
|
||||
done
|
||||
fail "$bucket_name_one and/or $bucket_name_two not listed (all buckets: ${bucket_array[*]})"
|
||||
delete_bucket $bucket_name_one || local deleted_one=$?
|
||||
[[ $deleted_one -eq 0 ]] || fail "Failed to delete bucket one"
|
||||
delete_bucket $bucket_name_two || local deleted_two=$?
|
||||
[[ $deleted_two -eq 0 ]] || fail "Failed to delete bucket one"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_bucket_or_contents "$BUCKET_TWO_NAME"
|
||||
|
||||
if [ $bucket_one_found != true ] || [ $bucket_two_found != true ]; then
|
||||
fail "'$BUCKET_ONE_NAME' and/or '$BUCKET_TWO_NAME' not listed (all buckets: ${bucket_array[*]})"
|
||||
fi
|
||||
}
|
||||
|
||||
# test listing a bucket's objects on versitygw
|
||||
@test "test_list_objects" {
|
||||
|
||||
bucket_name="versity-gwtest-list-object"
|
||||
object_one="test-file-one"
|
||||
object_two="test-file-two"
|
||||
|
||||
touch $object_one $object_two
|
||||
check_and_create_bucket $bucket_name || local result_one=$?
|
||||
create_test_files $object_one $object_two
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result_one=$?
|
||||
[[ result_one -eq 0 ]] || fail "Error creating bucket"
|
||||
put_object $object_one "$bucket_name"/"$object_one" || local result_two=$?
|
||||
put_object "$test_file_folder"/$object_one "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$?
|
||||
[[ result_two -eq 0 ]] || fail "Error adding object one"
|
||||
put_object $object_two "$bucket_name"/"$object_two" || local result_three=$?
|
||||
put_object "$test_file_folder"/$object_two "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
|
||||
[[ result_three -eq 0 ]] || fail "Error adding object two"
|
||||
list_objects $bucket_name
|
||||
|
||||
list_objects "$BUCKET_ONE_NAME"
|
||||
local object_one_found=false
|
||||
local object_two_found=false
|
||||
for object in "${object_array[@]}"; do
|
||||
@@ -129,44 +99,47 @@ source ./tests/util.sh
|
||||
object_two_found=true
|
||||
fi
|
||||
done
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $object_one $object_two
|
||||
|
||||
if [ $object_one_found != true ] || [ $object_two_found != true ]; then
|
||||
fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
|
||||
fi
|
||||
delete_object "$bucket_name"/"$object_one"
|
||||
delete_object "$bucket_name"/"$object_two"
|
||||
delete_bucket $bucket_name
|
||||
rm $object_one $object_two
|
||||
}
|
||||
|
||||
# test ability to retrieve bucket ACLs
|
||||
@test "test_get_bucket_acl" {
|
||||
|
||||
local bucket_name="versity-gwtest-get-bucket-acl"
|
||||
check_and_create_bucket $bucket_name || local created=$?
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating bucket"
|
||||
get_bucket_acl $bucket_name || local result=$?
|
||||
|
||||
get_bucket_acl "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Error retrieving acl"
|
||||
|
||||
id=$(echo "$acl" | jq '.Owner.ID')
|
||||
[[ $id == '"'"$AWS_ACCESS_KEY_ID"'"' ]] || fail "Acl mismatch"
|
||||
delete_bucket $bucket_name
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
}
|
||||
|
||||
# test ability to delete multiple objects from bucket
|
||||
@test "test_delete_objects" {
|
||||
|
||||
local bucket_name="versity-gwtest-delete-objects"
|
||||
local object_one="test-file-one"
|
||||
local object_two="test-file-two"
|
||||
|
||||
touch "$object_one" "$object_two"
|
||||
check_and_create_bucket $bucket_name || local result_one=$?
|
||||
create_test_files "$object_one" "$object_two" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result_one=$?
|
||||
[[ $result_one -eq 0 ]] || fail "Error creating bucket"
|
||||
put_object "$object_one" "$bucket_name"/"$object_one" || local result_two=$?
|
||||
|
||||
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$?
|
||||
[[ $result_two -eq 0 ]] || fail "Error adding object one"
|
||||
put_object "$object_two" "$bucket_name"/"$object_two" || local result_three=$?
|
||||
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
|
||||
[[ $result_three -eq 0 ]] || fail "Error adding object two"
|
||||
|
||||
error=$(aws s3api delete-objects --bucket $bucket_name --delete '{
|
||||
error=$(aws s3api delete-objects --bucket "$BUCKET_ONE_NAME" --delete '{
|
||||
"Objects": [
|
||||
{"Key": "test-file-one"},
|
||||
{"Key": "test-file-two"}
|
||||
@@ -174,11 +147,257 @@ source ./tests/util.sh
|
||||
}') || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Error deleting objects: $error"
|
||||
|
||||
object_exists "$bucket_name"/"$object_one" || local exists_one=$?
|
||||
object_exists "$BUCKET_ONE_NAME"/"$object_one" || local exists_one=$?
|
||||
[[ $exists_one -eq 1 ]] || fail "Object one not deleted"
|
||||
object_exists "$bucket_name"/"$object_two" || local exists_two=$?
|
||||
object_exists "$BUCKET_ONE_NAME"/"$object_two" || local exists_two=$?
|
||||
[[ $exists_two -eq 1 ]] || fail "Object two not deleted"
|
||||
|
||||
delete_bucket $bucket_name
|
||||
rm "$object_one" "$object_two"
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$object_one" "$object_two"
|
||||
}
|
||||
|
||||
# test abilty to set and retrieve bucket tags
|
||||
@test "test-set-get-bucket-tags" {
|
||||
|
||||
local key="test_key"
|
||||
local value="test_value"
|
||||
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
get_bucket_tags "$BUCKET_ONE_NAME" || local get_result=$?
|
||||
[[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
|
||||
tag_set=$(echo "$tags" | jq '.TagSet')
|
||||
[[ $tag_set == "[]" ]] || fail "Error: tags not empty"
|
||||
|
||||
put_bucket_tag "$BUCKET_ONE_NAME" $key $value
|
||||
get_bucket_tags "$BUCKET_ONE_NAME" || local get_result_two=$?
|
||||
[[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
|
||||
tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
|
||||
tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
|
||||
[[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
|
||||
[[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
}
|
||||
|
||||
# test v1 s3api list objects command
|
||||
@test "test-s3api-list-objects-v1" {
|
||||
|
||||
local object_one="test-file-one"
|
||||
local object_two="test-file-two"
|
||||
local object_two_data="test data\n"
|
||||
|
||||
create_test_files "$object_one" "$object_two" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
|
||||
[[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
|
||||
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
|
||||
[[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
|
||||
|
||||
list_objects_s3api_v1 "$BUCKET_ONE_NAME"
|
||||
key_one=$(echo "$objects" | jq '.Contents[0].Key')
|
||||
[[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
|
||||
size_one=$(echo "$objects" | jq '.Contents[0].Size')
|
||||
[[ $size_one -eq 0 ]] || fail "Object one size mismatch"
|
||||
key_two=$(echo "$objects" | jq '.Contents[1].Key')
|
||||
[[ $key_two == '"'$object_two'"' ]] || fail "Object two mismatch"
|
||||
size_two=$(echo "$objects" | jq '.Contents[1].Size')
|
||||
[[ $size_two -eq ${#object_two_data} ]] || fail "Object two size mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$object_one" "$object_two"
|
||||
}
|
||||
|
||||
# test v2 s3api list objects command
|
||||
@test "test-s3api-list-objects-v2" {
|
||||
|
||||
local object_one="test-file-one"
|
||||
local object_two="test-file-two"
|
||||
local object_two_data="test data\n"
|
||||
|
||||
create_test_files "$object_one" "$object_two" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
|
||||
[[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
|
||||
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
|
||||
[[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
|
||||
|
||||
list_objects_s3api_v2 "$BUCKET_ONE_NAME"
|
||||
key_one=$(echo "$objects" | jq '.Contents[0].Key')
|
||||
[[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
|
||||
size_one=$(echo "$objects" | jq '.Contents[0].Size')
|
||||
[[ $size_one -eq 0 ]] || fail "Object one size mismatch"
|
||||
key_two=$(echo "$objects" | jq '.Contents[1].Key')
|
||||
[[ $key_two == '"'$object_two'"' ]] || fail "Object two mismatch"
|
||||
size_two=$(echo "$objects" | jq '.Contents[1].Size')
|
||||
[[ $size_two -eq ${#object_two_data} ]] || fail "Object two size mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$object_one" "$object_two"
|
||||
}
|
||||
|
||||
# test abilty to set and retrieve object tags
|
||||
@test "test-set-get-object-tags" {
|
||||
|
||||
local bucket_file="bucket-file"
|
||||
local key="test_key"
|
||||
local value="test_value"
|
||||
|
||||
create_test_files "$bucket_file" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
|
||||
put_object "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
|
||||
[[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
|
||||
[[ $get_result -eq 0 ]] || fail "Error getting object tags"
|
||||
tag_set=$(echo "$tags" | jq '.TagSet')
|
||||
[[ $tag_set == "[]" ]] || fail "Error: tags not empty"
|
||||
|
||||
put_object_tag "$BUCKET_ONE_NAME" $bucket_file $key $value
|
||||
get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
|
||||
[[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
|
||||
tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
|
||||
tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
|
||||
[[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
|
||||
[[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $bucket_file
|
||||
}
|
||||
|
||||
# test multi-part upload
|
||||
@test "test-multi-part-upload" {
|
||||
|
||||
local bucket_file="bucket-file"
|
||||
bucket_file_data="test file\n"
|
||||
|
||||
create_test_files "$bucket_file" || local created=$?
|
||||
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$?
|
||||
[[ $upload_result -eq 0 ]] || fail "Error performing multipart upload"
|
||||
|
||||
copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy"
|
||||
copy_data=$(<"$test_file_folder"/$bucket_file-copy)
|
||||
[[ $bucket_file_data == "$copy_data" ]] || fail "Data doesn't match"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $bucket_file
|
||||
}
|
||||
|
||||
# test multi-part upload abort
|
||||
@test "test-multi-part-upload-abort" {
|
||||
|
||||
local bucket_file="bucket-file"
|
||||
bucket_file_data="test file\n"
|
||||
|
||||
create_test_files "$bucket_file" || local created=$?
|
||||
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$?
|
||||
[[ $abort_result -eq 0 ]] || fail "Abort failed"
|
||||
|
||||
object_exists "$BUCKET_ONE_NAME/$bucket_file" || exists=$?
|
||||
[[ $exists -eq 1 ]] || fail "Upload file exists after abort"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $bucket_file
|
||||
}
|
||||
|
||||
# test multi-part upload list parts command
|
||||
@test "test-multipart-upload-list-parts" {
|
||||
|
||||
local bucket_file="bucket-file"
|
||||
local bucket_file_data="test file\n"
|
||||
|
||||
create_test_files "$bucket_file" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
list_parts "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || list_result=$?
|
||||
[[ list_result -eq 0 ]] || fail "Listing multipart upload parts failed"
|
||||
|
||||
declare -a parts_map
|
||||
for ((i=0;i<$4;i++)) {
|
||||
local part_number
|
||||
local etag
|
||||
part_number=$(echo "$parts" | jq ".[$i].PartNumber")
|
||||
if [[ $part_number -eq "" ]]; then
|
||||
echo "error: blank part number"
|
||||
return 1
|
||||
fi
|
||||
etag=$(echo "$parts" | jq ".[$i].ETag")
|
||||
if [[ $etag == "" ]]; then
|
||||
echo "error: blank etag"
|
||||
return 1
|
||||
fi
|
||||
parts_map[$part_number]=$etag
|
||||
}
|
||||
|
||||
for ((i=0;i<$4;i++)) {
|
||||
local part_number
|
||||
local etag
|
||||
part_number=$(echo "$listed_parts" | jq ".Parts[$i].PartNumber")
|
||||
etag=$(echo "$listed_parts" | jq ".Parts[$i].ETag")
|
||||
if [[ ${parts_map[$part_number]} != "$etag" ]]; then
|
||||
echo "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $bucket_file
|
||||
}
|
||||
|
||||
# test listing of active uploads
|
||||
@test "test-multipart-upload-list-uploads" {
|
||||
|
||||
local bucket_file_one="bucket-file-one"
|
||||
local bucket_file_two="bucket-file-two"
|
||||
|
||||
create_test_files "$bucket_file_one" "$bucket_file_two" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
list_multipart_uploads "$BUCKET_ONE_NAME" "$test_file_folder"/"$bucket_file_one" "$test_file_folder"/"$bucket_file_two"
|
||||
[[ $? -eq 0 ]] || fail "failed to list multipart uploads"
|
||||
|
||||
local key_one
|
||||
local key_two
|
||||
key_one=$(echo "$uploads" | jq '.Uploads[0].Key')
|
||||
key_two=$(echo "$uploads" | jq '.Uploads[1].Key')
|
||||
key_one=${key_one//\"/}
|
||||
key_two=${key_two//\"/}
|
||||
echo "$test_file_folder/${bucket_file_one}abc"
|
||||
echo "${key_one}abc"
|
||||
echo "Length of test_file_folder/bucket_file_one: ${#test_file_folder}/${#bucket_file_one}"
|
||||
echo "Length of key_one: ${#key_one}"
|
||||
if [[ "$test_file_folder/$bucket_file_one" != *"$key_one" ]]; then
|
||||
fail "Key mismatch ($test_file_folder/$bucket_file_one, $key_one)"
|
||||
fi
|
||||
if [[ "$test_file_folder/$bucket_file_two" != *"$key_two" ]]; then
|
||||
fail "Key mismatch ($test_file_folder/$bucket_file_two, $key_two)"
|
||||
fi
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$bucket_file_one" "$bucket_file_two"
|
||||
}
|
||||
|
||||
@@ -15,7 +15,8 @@ setup() {
|
||||
fi
|
||||
else
|
||||
echo "$VERSITYGW_TEST_ENV"
|
||||
source $VERSITYGW_TEST_ENV
|
||||
# shellcheck source=./.env.default
|
||||
source "$VERSITYGW_TEST_ENV"
|
||||
fi
|
||||
|
||||
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
|
||||
@@ -42,6 +43,18 @@ setup() {
|
||||
elif [ -z "$AWS_ENDPOINT_URL" ]; then
|
||||
echo "No AWS endpoint URL set"
|
||||
return 1
|
||||
elif [ -z "$BUCKET_ONE_NAME" ]; then
|
||||
echo "No bucket one name set"
|
||||
return 1
|
||||
elif [ -z "$BUCKET_TWO_NAME" ]; then
|
||||
echo "No bucket two name set"
|
||||
return 1
|
||||
elif [ -z "$RECREATE_BUCKETS" ]; then
|
||||
echo "No recreate buckets parameter set"
|
||||
return 1
|
||||
elif [[ $RECREATE_BUCKETS != "true" ]] && [[ $RECREATE_BUCKETS != "false" ]]; then
|
||||
echo "RECREATE_BUCKETS must be 'true' or 'false'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
ROOT_ACCESS_KEY="$AWS_ACCESS_KEY_ID" ROOT_SECRET_KEY="$AWS_SECRET_ACCESS_KEY" "$VERSITY_EXE" "$BACKEND" "$LOCAL_FOLDER" &
|
||||
@@ -50,6 +63,8 @@ setup() {
|
||||
export AWS_PROFILE
|
||||
export AWS_ENDPOINT_URL
|
||||
export LOCAL_FOLDER
|
||||
export BUCKET_ONE_NAME
|
||||
export BUCKET_TWO_NAME
|
||||
|
||||
versitygw_pid=$!
|
||||
export versitygw_pid
|
||||
|
||||
471
tests/util.sh
471
tests/util.sh
@@ -1,3 +1,5 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
# create an AWS bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
@@ -6,6 +8,7 @@ create_bucket() {
|
||||
echo "create bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 mb s3://"$1" 2>&1) || exit_code=$?
|
||||
@@ -24,6 +27,7 @@ delete_bucket() {
|
||||
echo "delete bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rb s3://"$1" 2>&1) || exit_code="$?"
|
||||
@@ -38,6 +42,48 @@ delete_bucket() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# recursively delete an AWS bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
delete_bucket_recursive() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rb s3://"$1" --force 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
if [[ "$error" == *"The specified bucket does not exist"* ]]; then
|
||||
return 0
|
||||
else
|
||||
echo "error deleting bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# delete contents of a bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
delete_bucket_contents() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rm s3://"$1" --recursive 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error deleting bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if bucket exists
|
||||
# param: bucket name
|
||||
# return 0 for true, 1 for false, 2 for error
|
||||
@@ -46,6 +92,7 @@ bucket_exists() {
|
||||
echo "bucket exists check missing bucket name"
|
||||
return 2
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
|
||||
@@ -61,10 +108,35 @@ bucket_exists() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# create bucket if it doesn't exist
|
||||
# delete buckets or just the contents depending on RECREATE_BUCKETS parameter
|
||||
# param: bucket name
|
||||
# return: 0 for success, 1 for failure
|
||||
delete_bucket_or_contents() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete bucket or contents function requires bucket name"
|
||||
return 1
|
||||
fi
|
||||
if [[ $RECREATE_BUCKETS != "true" ]]; then
|
||||
delete_bucket_contents "$1" || local delete_result=$?
|
||||
if [[ $delete_result -ne 0 ]]; then
|
||||
echo "error deleting bucket contents"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
delete_bucket_recursive "$1" || local delete_result=$?
|
||||
if [[ $delete_result -ne 0 ]]; then
|
||||
echo "Bucket deletion error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# if RECREATE_BUCKETS is set to true create bucket, deleting it if it exists to clear state. If not,
|
||||
# check to see if it exists and return an error if it does not.
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
check_and_create_bucket() {
|
||||
setup_bucket() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket creation function requires bucket name"
|
||||
return 1
|
||||
@@ -75,14 +147,24 @@ check_and_create_bucket() {
|
||||
echo "Bucket existence check error"
|
||||
return 1
|
||||
fi
|
||||
local create_result
|
||||
if [[ $exists_result -eq 1 ]]; then
|
||||
create_bucket "$1" || create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
echo "Error creating bucket"
|
||||
if [[ $exists_result -eq 0 ]]; then
|
||||
delete_bucket_or_contents "$1" || delete_result=$?
|
||||
if [[ delete_result -ne 0 ]]; then
|
||||
echo "error deleting bucket or contents"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
if [[ $RECREATE_BUCKETS != "true" ]]; then
|
||||
echo "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
|
||||
return 1
|
||||
fi
|
||||
local create_result
|
||||
create_bucket "$1" || create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
echo "Error creating bucket"
|
||||
return 1
|
||||
fi
|
||||
echo "Bucket creation success"
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -135,13 +217,13 @@ check_and_put_object() {
|
||||
return 1
|
||||
fi
|
||||
object_exists "$2" || local exists_result=$?
|
||||
if [ $exists_result -eq 2 ]; then
|
||||
if [ "$exists_result" -eq 2 ]; then
|
||||
echo "error checking if object exists"
|
||||
return 1
|
||||
fi
|
||||
if [ $exists_result -eq 1 ]; then
|
||||
if [ "$exists_result" -eq 1 ]; then
|
||||
put_object "$1" "$2" || local put_result=$?
|
||||
if [ $put_result -ne 0 ]; then
|
||||
if [ "$put_result" -ne 0 ]; then
|
||||
echo "error adding object"
|
||||
return 1
|
||||
fi
|
||||
@@ -150,7 +232,7 @@ check_and_put_object() {
|
||||
}
|
||||
|
||||
# delete object from versitygw
|
||||
# param: object location
|
||||
# param: object path, including bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
delete_object() {
|
||||
if [ $# -ne 1 ]; then
|
||||
@@ -266,8 +348,373 @@ get_bucket_acl() {
|
||||
local exit_code=0
|
||||
acl=$(aws s3api get-bucket-acl --bucket "$1" 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "Error: $acl"
|
||||
echo "Error getting bucket ACLs: $acl"
|
||||
return 1
|
||||
fi
|
||||
export acl
|
||||
}
|
||||
|
||||
# add tags to bucket
|
||||
# params: bucket, key, value
|
||||
# return: 0 for success, 1 for error
|
||||
put_bucket_tag() {
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "bucket tag command missing bucket name, key, value"
|
||||
return 1
|
||||
fi
|
||||
local error
|
||||
local result
|
||||
error=$(aws s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "Error adding bucket tag: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# get bucket tags
|
||||
# params: bucket
|
||||
# export 'tags' on success, return 1 for error
|
||||
get_bucket_tags() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "get bucket tag command missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
local result
|
||||
tags=$(aws s3api get-bucket-tagging --bucket "$1") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error getting bucket tags: $tags"
|
||||
return 1
|
||||
fi
|
||||
export tags
|
||||
}
|
||||
|
||||
# add tags to object
|
||||
# params: object, key, value
|
||||
# return: 0 for success, 1 for error
|
||||
put_object_tag() {
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "object tag command missing object name, file, key, and/or value"
|
||||
return 1
|
||||
fi
|
||||
local error
|
||||
local result
|
||||
error=$(aws s3api put-object-tagging --bucket "$1" --key "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "Error adding object tag: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# get object tags
|
||||
# params: bucket
|
||||
# export 'tags' on success, return 1 for error
|
||||
get_object_tags() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "get object tag command missing bucket and/or key"
|
||||
return 1
|
||||
fi
|
||||
local result
|
||||
tags=$(aws s3api get-object-tagging --bucket "$1" --key "$2") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error getting object tags: $tags"
|
||||
return 1
|
||||
fi
|
||||
export tags
|
||||
}
|
||||
|
||||
# create a test file and export folder. do so in temp folder
|
||||
# params: filename
|
||||
# export test file folder on success, return 1 for error
|
||||
create_test_files() {
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "create test files command missing filename"
|
||||
return 1
|
||||
fi
|
||||
test_file_folder=.
|
||||
if [[ -z "$GITHUB_ACTIONS" ]]; then
|
||||
test_file_folder=${TMPDIR}versity-gwtest
|
||||
mkdir -p "$test_file_folder" || local mkdir_result=$?
|
||||
if [[ $mkdir_result -ne 0 ]]; then
|
||||
echo "error creating test file folder"
|
||||
fi
|
||||
fi
|
||||
for name in "$@"; do
|
||||
touch "$test_file_folder"/"$name" || local touch_result=$?
|
||||
if [[ $touch_result -ne 0 ]]; then
|
||||
echo "error creating file $name"
|
||||
fi
|
||||
done
|
||||
export test_file_folder
|
||||
}
|
||||
|
||||
# delete a test file
|
||||
# params: filename
|
||||
# return: 0 for success, 1 for error
|
||||
delete_test_files() {
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "delete test files command missing filenames"
|
||||
return 1
|
||||
fi
|
||||
if [ -z "$test_file_folder" ]; then
|
||||
echo "no test file folder defined, not deleting"
|
||||
return 1
|
||||
fi
|
||||
for name in "$@"; do
|
||||
rm "$test_file_folder"/"$name" || rm_result=$?
|
||||
if [[ $rm_result -ne 0 ]]; then
|
||||
echo "error deleting file $name"
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# list objects in bucket, v1
|
||||
# param: bucket
|
||||
# export objects on success, return 1 for failure
|
||||
list_objects_s3api_v1() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "list objects command missing bucket"
|
||||
return 1
|
||||
fi
|
||||
objects=$(aws s3api list-objects --bucket "$1") || local result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error listing objects: $objects"
|
||||
return 1
|
||||
fi
|
||||
export objects
|
||||
}
|
||||
|
||||
# list objects in bucket, v2
|
||||
# param: bucket
|
||||
# export objects on success, return 1 for failure
|
||||
list_objects_s3api_v2() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "list objects command missing bucket and/or path"
|
||||
return 1
|
||||
fi
|
||||
objects=$(aws s3api list-objects-v2 --bucket "$1") || local result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error listing objects: $objects"
|
||||
return 1
|
||||
fi
|
||||
export objects
|
||||
}
|
||||
|
||||
# initialize a multipart upload
|
||||
# params: bucket, key
|
||||
# return 0 for success, 1 for failure
|
||||
create_multipart_upload() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "create multipart upload function must have bucket, key"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local multipart_data
|
||||
multipart_data=$(aws s3api create-multipart-upload --bucket "$1" --key "$2") || local created=$?
|
||||
if [[ $created -ne 0 ]]; then
|
||||
echo "Error creating multipart upload: $upload_id"
|
||||
return 1
|
||||
fi
|
||||
|
||||
upload_id=$(echo "$multipart_data" | jq '.UploadId')
|
||||
upload_id="${upload_id//\"/}"
|
||||
export upload_id
|
||||
}
|
||||
|
||||
# upload a single part of a multipart upload
|
||||
# params: bucket, key, upload ID, original (unsplit) file name, part number
|
||||
# return: 0 for success, 1 for failure
|
||||
upload_part() {
|
||||
if [ $# -ne 5 ]; then
|
||||
echo "upload multipart part function must have bucket, key, upload ID, file name, part number"
|
||||
return 1
|
||||
fi
|
||||
local etag_json
|
||||
etag_json=$(aws s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))") || local uploaded=$?
|
||||
if [[ $uploaded -ne 0 ]]; then
|
||||
echo "Error uploading part $5: $etag_json"
|
||||
return 1
|
||||
fi
|
||||
etag=$(echo "$etag_json" | jq '.ETag')
|
||||
export etag
|
||||
}
|
||||
|
||||
# perform all parts of a multipart upload before completion command
|
||||
# params: bucket, key, file to split and upload, number of file parts to upload
|
||||
# return: 0 for success, 1 for failure
|
||||
multipart_upload_before_completion() {
|
||||
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "multipart upload pre-completion command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
file_size=$(stat -c %s "$3" 2>/dev/null || stat -f %z "$3" 2>/dev/null)
|
||||
part_size=$((file_size / $4))
|
||||
remainder=$((file_size % $4))
|
||||
if [[ remainder -ne 0 ]]; then
|
||||
part_size=$((part_size+1))
|
||||
fi
|
||||
local error
|
||||
local split_result
|
||||
error=$(split -a 1 -d -b "$part_size" "$3" "$3"-) || split_result=$?
|
||||
if [[ $split_result -ne 0 ]]; then
|
||||
echo "error splitting file: $error"
|
||||
return 1
|
||||
fi
|
||||
|
||||
create_multipart_upload "$1" "$2" || create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
echo "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
upload_part "$1" "$2" "$upload_id" "$3" "$i" || local upload_result=$?
|
||||
if [[ $upload_result -ne 0 ]]; then
|
||||
echo "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
# perform a multi-part upload
|
||||
# params: bucket, key, source file location, number of parts
|
||||
# return 0 for success, 1 for failure
|
||||
multipart_upload() {
|
||||
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "multipart upload command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
error=$(aws s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}') || local completed=$?
|
||||
if [[ $completed -ne 0 ]]; then
|
||||
echo "Error completing upload: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# run the abort multipart command
|
||||
# params: bucket, key, upload ID
|
||||
# return 0 for success, 1 for failure
|
||||
run_abort_command() {
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "command to run abort requires bucket, key, upload ID"
|
||||
return 1
|
||||
fi
|
||||
|
||||
error=$(aws s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3") || local aborted=$?
|
||||
if [[ $aborted -ne 0 ]]; then
|
||||
echo "Error aborting upload: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# run upload, then abort it
|
||||
# params: bucket, key, local file location, number of parts to split into before uploading
|
||||
# return 0 for success, 1 for failure
|
||||
abort_multipart_upload() {
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "abort multipart upload command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
run_abort_command "$1" "$2" "$upload_id"
|
||||
return $?
|
||||
}
|
||||
|
||||
# copy a file to/from S3
|
||||
# params: source, destination
|
||||
# return 0 for success, 1 for failure
|
||||
copy_file() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "copy file command requires src and dest"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local result
|
||||
error=$(aws s3 cp "$1" "$2") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error copying file: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# list parts of an unfinished multipart upload
|
||||
# params: bucket, key, local file location, and parts to split into before upload
|
||||
# export parts on success, return 1 for error
|
||||
list_parts() {
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "list multipart upload parts command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
listed_parts=$(aws s3api list-parts --bucket "$1" --key "$2" --upload-id "$upload_id") || local listed=$?
|
||||
if [[ $listed -ne 0 ]]; then
|
||||
echo "Error aborting upload: $parts"
|
||||
return 1
|
||||
fi
|
||||
export listed_parts
|
||||
}
|
||||
|
||||
# list unfinished multipart uploads
|
||||
# params: bucket, key one, key two
|
||||
# export current two uploads on success, return 1 for error
|
||||
list_multipart_uploads() {
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "list multipart uploads command requires bucket and two keys"
|
||||
return 1
|
||||
fi
|
||||
|
||||
create_multipart_upload "$1" "$2" || local create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
echo "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
create_multipart_upload "$1" "$3" || local create_result_two=$?
|
||||
if [[ $create_result_two -ne 0 ]]; then
|
||||
echo "error creating multpart upload two"
|
||||
return 1
|
||||
fi
|
||||
|
||||
uploads=$(aws s3api list-multipart-uploads --bucket "$1") || local list_result=$?
|
||||
if [[ $list_result -ne 0 ]]; then
|
||||
echo "error listing uploads: $uploads"
|
||||
return 1
|
||||
fi
|
||||
export uploads
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
|
||||
#!/usr/bin/env bats
|
||||
|
||||
# check if object exists both on S3 and locally
|
||||
# param: object path
|
||||
|
||||
Reference in New Issue
Block a user