mirror of
https://github.com/versity/versitygw.git
synced 2026-02-02 08:22:03 +00:00
Compare commits
29 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
905b283421 | ||
|
|
6fea34acda | ||
|
|
1c29fbfd81 | ||
|
|
a3b14d3a05 | ||
|
|
cafb57eb33 | ||
|
|
0760467c3d | ||
|
|
4d168da376 | ||
|
|
cde033811f | ||
|
|
7a56c7e15e | ||
|
|
e21e514997 | ||
|
|
660709fe6d | ||
|
|
5931d713f2 | ||
|
|
08e5eb02a0 | ||
|
|
7cba952546 | ||
|
|
5d6c0f8b67 | ||
|
|
be17b3fd33 | ||
|
|
e6440da30a | ||
|
|
443da7f9a4 | ||
|
|
6c56307746 | ||
|
|
9765eadd84 | ||
|
|
4619171f86 | ||
|
|
89b4b615ab | ||
|
|
0c056f935b | ||
|
|
bf1e2c83d5 | ||
|
|
68794518af | ||
|
|
3cce3a5201 | ||
|
|
d70ea61830 | ||
|
|
9d0cf77b25 | ||
|
|
0d3a238ceb |
45
.github/workflows/docker.yaml
vendored
Normal file
45
.github/workflows/docker.yaml
vendored
Normal file
@@ -0,0 +1,45 @@
|
||||
name: Publish Docker image
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
push_to_registries:
|
||||
name: Push Docker image to multiple registries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
versity/versitygw
|
||||
ghcr.io/${{ github.repository }}
|
||||
|
||||
- name: Build and push Docker images
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
16
.github/workflows/system.yml
vendored
16
.github/workflows/system.yml
vendored
@@ -1,19 +1,19 @@
|
||||
name: system tests
|
||||
on: pull_request
|
||||
#on:
|
||||
# workflow_dispatch:
|
||||
# inputs:
|
||||
# run_workflow:
|
||||
# description: 'Run command-line tests'
|
||||
jobs:
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Install ShellCheck
|
||||
run: sudo apt-get install shellcheck
|
||||
|
||||
- name: Run ShellCheck
|
||||
run: shellcheck -S warning ./tests/*.sh
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
@@ -38,5 +38,5 @@ jobs:
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
|
||||
export VERSITY_EXE=./versitygw
|
||||
mkdir /tmp/gw
|
||||
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/s3_bucket_tests.sh
|
||||
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.versitygw $HOME/bin/bats ./tests/posix_tests.sh
|
||||
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.default $HOME/bin/bats ./tests/s3_bucket_tests.sh
|
||||
VERSITYGW_TEST_ENV=$GITHUB_WORKSPACE/tests/.env.default $HOME/bin/bats ./tests/posix_tests.sh
|
||||
|
||||
@@ -27,6 +27,7 @@ archives:
|
||||
# this name template makes the OS and Arch compatible with the results of uname.
|
||||
name_template: >-
|
||||
{{ .ProjectName }}_
|
||||
v{{ .Version }}_
|
||||
{{- title .Os }}_
|
||||
{{- if eq .Arch "amd64" }}x86_64
|
||||
{{- else if eq .Arch "386" }}i386
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.20-alpine
|
||||
FROM golang:latest
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -8,6 +8,7 @@ RUN go mod download
|
||||
COPY ./ ./
|
||||
|
||||
WORKDIR /app/cmd/versitygw
|
||||
ENV CGO_ENABLED=0
|
||||
RUN go build -o versitygw
|
||||
|
||||
FROM alpine:latest
|
||||
|
||||
@@ -65,6 +65,11 @@ type Backend interface {
|
||||
RestoreObject(context.Context, *s3.RestoreObjectInput) error
|
||||
SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
|
||||
|
||||
// bucket tagging operations
|
||||
GetBucketTagging(_ context.Context, bucket string) (map[string]string, error)
|
||||
PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error
|
||||
DeleteBucketTagging(_ context.Context, bucket string) error
|
||||
|
||||
// object tags operations
|
||||
GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
|
||||
PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
|
||||
@@ -179,6 +184,16 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
|
||||
}
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucketTagging(_ context.Context, bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
@@ -1640,7 +1640,15 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
|
||||
}
|
||||
marker := ""
|
||||
if input.ContinuationToken != nil {
|
||||
marker = *input.ContinuationToken
|
||||
if input.StartAfter != nil {
|
||||
if *input.StartAfter > *input.ContinuationToken {
|
||||
marker = *input.StartAfter
|
||||
} else {
|
||||
marker = *input.ContinuationToken
|
||||
}
|
||||
} else {
|
||||
marker = *input.ContinuationToken
|
||||
}
|
||||
}
|
||||
delim := ""
|
||||
if input.Delimiter != nil {
|
||||
@@ -1720,6 +1728,57 @@ func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (p *Posix) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
if tags == nil {
|
||||
err = xattr.Remove(bucket, "user."+tagHdr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove tags: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
b, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal tags: %w", err)
|
||||
}
|
||||
|
||||
err = xattr.Set(bucket, "user."+tagHdr, b)
|
||||
if err != nil {
|
||||
return fmt.Errorf("set tags: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Posix) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
tags, err := p.getXattrTags(bucket, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func (p *Posix) DeleteBucketTagging(ctx context.Context, bucket string) error {
|
||||
return p.PutBucketTagging(ctx, bucket, nil)
|
||||
}
|
||||
|
||||
func (p *Posix) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
|
||||
@@ -268,11 +268,11 @@ func getAction(tf testFunc) func(*cli.Context) error {
|
||||
func extractIntTests() (commands []*cli.Command) {
|
||||
tests := integration.GetIntTests()
|
||||
for key, val := range tests {
|
||||
testKey := key
|
||||
k := key
|
||||
testFunc := val
|
||||
commands = append(commands, &cli.Command{
|
||||
Name: testKey,
|
||||
Usage: fmt.Sprintf("Runs %v integration test", testKey),
|
||||
Name: k,
|
||||
Usage: fmt.Sprintf("Runs %v integration test", key),
|
||||
Action: func(ctx *cli.Context) error {
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
|
||||
18
go.mod
18
go.mod
@@ -3,9 +3,9 @@ module github.com/versity/versitygw
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
|
||||
github.com/aws/smithy-go v1.19.0
|
||||
@@ -16,13 +16,13 @@ require (
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/segmentio/kafka-go v0.4.47
|
||||
github.com/urfave/cli/v2 v2.27.1
|
||||
github.com/valyala/fasthttp v1.51.0
|
||||
github.com/valyala/fasthttp v1.52.0
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
|
||||
golang.org/x/sys v0.16.0
|
||||
golang.org/x/sys v0.17.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
|
||||
@@ -38,13 +38,13 @@ require (
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.18 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
golang.org/x/crypto v0.18.0 // indirect
|
||||
golang.org/x/net v0.20.0 // indirect
|
||||
golang.org/x/crypto v0.19.0 // indirect
|
||||
golang.org/x/net v0.21.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.16
|
||||
@@ -57,7 +57,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/klauspost/compress v1.17.2 // indirect
|
||||
github.com/klauspost/compress v1.17.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
|
||||
36
go.sum
36
go.sum
@@ -1,20 +1,20 @@
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 h1:IfFdxTUDiV58iZqPKgyWiz4X4fCxZeQ1pTQPImLYXpY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
|
||||
@@ -76,8 +76,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
|
||||
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
@@ -118,8 +118,8 @@ github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
|
||||
github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
|
||||
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
|
||||
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
|
||||
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
|
||||
@@ -137,8 +137,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
||||
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
|
||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@@ -147,8 +147,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
|
||||
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
|
||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -165,8 +165,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
|
||||
@@ -22,10 +22,36 @@ func TestAuthentication(s *S3Conf) {
|
||||
Authentication_signature_error_incorrect_secret_key(s)
|
||||
}
|
||||
|
||||
func TestPresignedAuthentication(s *S3Conf) {
|
||||
PresignedAuth_missing_algo_query_param(s)
|
||||
PresignedAuth_unsupported_algorithm(s)
|
||||
PresignedAuth_missing_credentials_query_param(s)
|
||||
PresignedAuth_malformed_creds_invalid_parts(s)
|
||||
PresignedAuth_malformed_creds_invalid_parts(s)
|
||||
PresignedAuth_creds_incorrect_service(s)
|
||||
PresignedAuth_creds_incorrect_region(s)
|
||||
PresignedAuth_creds_invalid_date(s)
|
||||
PresignedAuth_missing_date_query(s)
|
||||
PresignedAuth_dates_mismatch(s)
|
||||
PresignedAuth_non_existing_access_key_id(s)
|
||||
PresignedAuth_missing_signed_headers_query_param(s)
|
||||
PresignedAuth_missing_expiration_query_param(s)
|
||||
PresignedAuth_invalid_expiration_query_param(s)
|
||||
PresignedAuth_negative_expiration_query_param(s)
|
||||
PresignedAuth_exceeding_expiration_query_param(s)
|
||||
PresignedAuth_expired_request(s)
|
||||
PresignedAuth_incorrect_secret_key(s)
|
||||
PresignedAuth_PutObject_success(s)
|
||||
PresignedAuth_Put_GetObject_with_data(s)
|
||||
PresignedAuth_UploadPart(s)
|
||||
}
|
||||
|
||||
func TestCreateBucket(s *S3Conf) {
|
||||
CreateBucket_invalid_bucket_name(s)
|
||||
CreateBucket_existing_bucket(s)
|
||||
CreateBucket_as_user(s)
|
||||
CreateBucket_default_acl(s)
|
||||
CreateBucket_non_default_acl(s)
|
||||
CreateDeleteBucket_success(s)
|
||||
}
|
||||
|
||||
@@ -46,6 +72,23 @@ func TestDeleteBucket(s *S3Conf) {
|
||||
DeleteBucket_success_status_code(s)
|
||||
}
|
||||
|
||||
func TestPutBucketTagging(s *S3Conf) {
|
||||
PutBucketTagging_non_existing_bucket(s)
|
||||
PutBucketTagging_long_tags(s)
|
||||
PutBucketTagging_success(s)
|
||||
}
|
||||
|
||||
func TestGetBucketTagging(s *S3Conf) {
|
||||
GetBucketTagging_non_existing_bucket(s)
|
||||
GetBucketTagging_success(s)
|
||||
}
|
||||
|
||||
func TestDeleteBucketTagging(s *S3Conf) {
|
||||
DeleteBucketTagging_non_existing_object(s)
|
||||
DeleteBucketTagging_success_status(s)
|
||||
DeleteBucketTagging_success(s)
|
||||
}
|
||||
|
||||
func TestPutObject(s *S3Conf) {
|
||||
PutObject_non_existing_bucket(s)
|
||||
PutObject_special_chars(s)
|
||||
@@ -78,6 +121,13 @@ func TestListObjects(s *S3Conf) {
|
||||
ListObjects_marker_not_from_obj_list(s)
|
||||
}
|
||||
|
||||
func TestListObjectsV2(s *S3Conf) {
|
||||
ListObjectsV2_start_after(s)
|
||||
ListObjectsV2_both_start_after_and_continuation_token(s)
|
||||
ListObjectsV2_start_after_not_in_list(s)
|
||||
ListObjectsV2_start_after_empty_result(s)
|
||||
}
|
||||
|
||||
func TestDeleteObject(s *S3Conf) {
|
||||
DeleteObject_non_existing_object(s)
|
||||
DeleteObject_success(s)
|
||||
@@ -193,14 +243,19 @@ func TestGetBucketAcl(s *S3Conf) {
|
||||
|
||||
func TestFullFlow(s *S3Conf) {
|
||||
TestAuthentication(s)
|
||||
TestPresignedAuthentication(s)
|
||||
TestCreateBucket(s)
|
||||
TestHeadBucket(s)
|
||||
TestListBuckets(s)
|
||||
TestDeleteBucket(s)
|
||||
TestPutBucketTagging(s)
|
||||
TestGetBucketTagging(s)
|
||||
TestDeleteBucketTagging(s)
|
||||
TestPutObject(s)
|
||||
TestHeadObject(s)
|
||||
TestGetObject(s)
|
||||
TestListObjects(s)
|
||||
TestListObjectsV2(s)
|
||||
TestDeleteObject(s)
|
||||
TestDeleteObjects(s)
|
||||
TestCopyObject(s)
|
||||
@@ -228,128 +283,162 @@ type IntTests map[string]func(s *S3Conf) error
|
||||
|
||||
func GetIntTests() IntTests {
|
||||
return IntTests{
|
||||
"Authentication_empty_auth_header": Authentication_empty_auth_header,
|
||||
"Authentication_invalid_auth_header": Authentication_invalid_auth_header,
|
||||
"Authentication_unsupported_signature_version": Authentication_unsupported_signature_version,
|
||||
"Authentication_malformed_credentials": Authentication_malformed_credentials,
|
||||
"Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts,
|
||||
"Authentication_credentials_terminated_string": Authentication_credentials_terminated_string,
|
||||
"Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service,
|
||||
"Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region,
|
||||
"Authentication_credentials_invalid_date": Authentication_credentials_invalid_date,
|
||||
"Authentication_credentials_future_date": Authentication_credentials_future_date,
|
||||
"Authentication_credentials_past_date": Authentication_credentials_past_date,
|
||||
"Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key,
|
||||
"Authentication_invalid_signed_headers": Authentication_invalid_signed_headers,
|
||||
"Authentication_missing_date_header": Authentication_missing_date_header,
|
||||
"Authentication_invalid_date_header": Authentication_invalid_date_header,
|
||||
"Authentication_date_mismatch": Authentication_date_mismatch,
|
||||
"Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash,
|
||||
"Authentication_incorrect_md5": Authentication_incorrect_md5,
|
||||
"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
|
||||
"CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name,
|
||||
"CreateBucket_existing_bucket": CreateBucket_existing_bucket,
|
||||
"CreateBucket_as_user": CreateBucket_as_user,
|
||||
"CreateDeleteBucket_success": CreateDeleteBucket_success,
|
||||
"HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket,
|
||||
"HeadBucket_success": HeadBucket_success,
|
||||
"ListBuckets_as_user": ListBuckets_as_user,
|
||||
"ListBuckets_as_admin": ListBuckets_as_admin,
|
||||
"ListBuckets_success": ListBuckets_success,
|
||||
"DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket,
|
||||
"DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket,
|
||||
"DeleteBucket_success_status_code": DeleteBucket_success_status_code,
|
||||
"PutObject_non_existing_bucket": PutObject_non_existing_bucket,
|
||||
"PutObject_special_chars": PutObject_special_chars,
|
||||
"PutObject_invalid_long_tags": PutObject_invalid_long_tags,
|
||||
"PutObject_success": PutObject_success,
|
||||
"PutObject_invalid_credentials": PutObject_invalid_credentials,
|
||||
"HeadObject_non_existing_object": HeadObject_non_existing_object,
|
||||
"HeadObject_success": HeadObject_success,
|
||||
"GetObject_non_existing_key": GetObject_non_existing_key,
|
||||
"GetObject_invalid_ranges": GetObject_invalid_ranges,
|
||||
"GetObject_with_meta": GetObject_with_meta,
|
||||
"GetObject_success": GetObject_success,
|
||||
"GetObject_by_range_success": GetObject_by_range_success,
|
||||
"ListObjects_non_existing_bucket": ListObjects_non_existing_bucket,
|
||||
"ListObjects_with_prefix": ListObjects_with_prefix,
|
||||
"ListObject_truncated": ListObject_truncated,
|
||||
"ListObjects_invalid_max_keys": ListObjects_invalid_max_keys,
|
||||
"ListObjects_max_keys_0": ListObjects_max_keys_0,
|
||||
"ListObjects_delimiter": ListObjects_delimiter,
|
||||
"ListObjects_max_keys_none": ListObjects_max_keys_none,
|
||||
"ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list,
|
||||
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
|
||||
"DeleteObject_success": DeleteObject_success,
|
||||
"DeleteObject_success_status_code": DeleteObject_success_status_code,
|
||||
"DeleteObjects_empty_input": DeleteObjects_empty_input,
|
||||
"DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects,
|
||||
"DeleteObjects_success": DeleteObjects_success,
|
||||
"CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket,
|
||||
"CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket,
|
||||
"CopyObject_copy_to_itself": CopyObject_copy_to_itself,
|
||||
"CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata,
|
||||
"CopyObject_success": CopyObject_success,
|
||||
"PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object,
|
||||
"PutObjectTagging_long_tags": PutObjectTagging_long_tags,
|
||||
"PutObjectTagging_success": PutObjectTagging_success,
|
||||
"GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object,
|
||||
"GetObjectTagging_success": GetObjectTagging_success,
|
||||
"DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object,
|
||||
"DeleteObjectTagging_success_status": DeleteObjectTagging_success_status,
|
||||
"DeleteObjectTagging_success": DeleteObjectTagging_success,
|
||||
"CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket,
|
||||
"CreateMultipartUpload_success": CreateMultipartUpload_success,
|
||||
"UploadPart_non_existing_bucket": UploadPart_non_existing_bucket,
|
||||
"UploadPart_invalid_part_number": UploadPart_invalid_part_number,
|
||||
"UploadPart_non_existing_key": UploadPart_non_existing_key,
|
||||
"UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload,
|
||||
"UploadPart_success": UploadPart_success,
|
||||
"UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket,
|
||||
"UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId,
|
||||
"UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key,
|
||||
"UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number,
|
||||
"UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source,
|
||||
"UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket,
|
||||
"UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key,
|
||||
"UploadPartCopy_success": UploadPartCopy_success,
|
||||
"UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range,
|
||||
"UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size,
|
||||
"UploadPartCopy_by_range_success": UploadPartCopy_by_range_success,
|
||||
"ListParts_incorrect_uploadId": ListParts_incorrect_uploadId,
|
||||
"ListParts_incorrect_object_key": ListParts_incorrect_object_key,
|
||||
"ListParts_success": ListParts_success,
|
||||
"ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket,
|
||||
"ListMultipartUploads_empty_result": ListMultipartUploads_empty_result,
|
||||
"ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads,
|
||||
"ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads,
|
||||
"ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker,
|
||||
"ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker,
|
||||
"ListMultipartUploads_success": ListMultipartUploads_success,
|
||||
"AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket,
|
||||
"AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId,
|
||||
"AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key,
|
||||
"AbortMultipartUpload_success": AbortMultipartUpload_success,
|
||||
"AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code,
|
||||
"CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket,
|
||||
"CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number,
|
||||
"CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag,
|
||||
"CompleteMultipartUpload_success": CompleteMultipartUpload_success,
|
||||
"PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket,
|
||||
"PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp,
|
||||
"PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants,
|
||||
"PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants,
|
||||
"PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner,
|
||||
"PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied,
|
||||
"PutBucketAcl_success_grants": PutBucketAcl_success_grants,
|
||||
"PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl,
|
||||
"PutBucketAcl_success_acp": PutBucketAcl_success_acp,
|
||||
"GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket,
|
||||
"GetBucketAcl_access_denied": GetBucketAcl_access_denied,
|
||||
"GetBucketAcl_success": GetBucketAcl_success,
|
||||
"PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj,
|
||||
"PutObject_overwrite_file_obj": PutObject_overwrite_file_obj,
|
||||
"PutObject_dir_obj_with_data": PutObject_dir_obj_with_data,
|
||||
"CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj,
|
||||
"Authentication_empty_auth_header": Authentication_empty_auth_header,
|
||||
"Authentication_invalid_auth_header": Authentication_invalid_auth_header,
|
||||
"Authentication_unsupported_signature_version": Authentication_unsupported_signature_version,
|
||||
"Authentication_malformed_credentials": Authentication_malformed_credentials,
|
||||
"Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts,
|
||||
"Authentication_credentials_terminated_string": Authentication_credentials_terminated_string,
|
||||
"Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service,
|
||||
"Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region,
|
||||
"Authentication_credentials_invalid_date": Authentication_credentials_invalid_date,
|
||||
"Authentication_credentials_future_date": Authentication_credentials_future_date,
|
||||
"Authentication_credentials_past_date": Authentication_credentials_past_date,
|
||||
"Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key,
|
||||
"Authentication_invalid_signed_headers": Authentication_invalid_signed_headers,
|
||||
"Authentication_missing_date_header": Authentication_missing_date_header,
|
||||
"Authentication_invalid_date_header": Authentication_invalid_date_header,
|
||||
"Authentication_date_mismatch": Authentication_date_mismatch,
|
||||
"Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash,
|
||||
"Authentication_incorrect_md5": Authentication_incorrect_md5,
|
||||
"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
|
||||
"PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param,
|
||||
"PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm,
|
||||
"PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param,
|
||||
"PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts,
|
||||
"PresignedAuth_creds_invalid_terminator": PresignedAuth_creds_invalid_terminator,
|
||||
"PresignedAuth_creds_incorrect_service": PresignedAuth_creds_incorrect_service,
|
||||
"PresignedAuth_creds_incorrect_region": PresignedAuth_creds_incorrect_region,
|
||||
"PresignedAuth_creds_invalid_date": PresignedAuth_creds_invalid_date,
|
||||
"PresignedAuth_missing_date_query": PresignedAuth_missing_date_query,
|
||||
"PresignedAuth_dates_mismatch": PresignedAuth_dates_mismatch,
|
||||
"PresignedAuth_non_existing_access_key_id": PresignedAuth_non_existing_access_key_id,
|
||||
"PresignedAuth_missing_signed_headers_query_param": PresignedAuth_missing_signed_headers_query_param,
|
||||
"PresignedAuth_missing_expiration_query_param": PresignedAuth_missing_expiration_query_param,
|
||||
"PresignedAuth_invalid_expiration_query_param": PresignedAuth_invalid_expiration_query_param,
|
||||
"PresignedAuth_negative_expiration_query_param": PresignedAuth_negative_expiration_query_param,
|
||||
"PresignedAuth_exceeding_expiration_query_param": PresignedAuth_exceeding_expiration_query_param,
|
||||
"PresignedAuth_expired_request": PresignedAuth_expired_request,
|
||||
"PresignedAuth_incorrect_secret_key": PresignedAuth_incorrect_secret_key,
|
||||
"PresignedAuth_PutObject_success": PresignedAuth_PutObject_success,
|
||||
"PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data,
|
||||
"PresignedAuth_UploadPart": PresignedAuth_UploadPart,
|
||||
"CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name,
|
||||
"CreateBucket_existing_bucket": CreateBucket_existing_bucket,
|
||||
"CreateBucket_as_user": CreateBucket_as_user,
|
||||
"CreateDeleteBucket_success": CreateDeleteBucket_success,
|
||||
"CreateBucket_default_acl": CreateBucket_default_acl,
|
||||
"CreateBucket_non_default_acl": CreateBucket_non_default_acl,
|
||||
"HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket,
|
||||
"HeadBucket_success": HeadBucket_success,
|
||||
"ListBuckets_as_user": ListBuckets_as_user,
|
||||
"ListBuckets_as_admin": ListBuckets_as_admin,
|
||||
"ListBuckets_success": ListBuckets_success,
|
||||
"DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket,
|
||||
"DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket,
|
||||
"DeleteBucket_success_status_code": DeleteBucket_success_status_code,
|
||||
"PutBucketTagging_non_existing_bucket": PutBucketTagging_non_existing_bucket,
|
||||
"PutBucketTagging_long_tags": PutBucketTagging_long_tags,
|
||||
"PutBucketTagging_success": PutBucketTagging_success,
|
||||
"GetBucketTagging_non_existing_bucket": GetBucketTagging_non_existing_bucket,
|
||||
"GetBucketTagging_success": GetBucketTagging_success,
|
||||
"DeleteBucketTagging_non_existing_object": DeleteBucketTagging_non_existing_object,
|
||||
"DeleteBucketTagging_success_status": DeleteBucketTagging_success_status,
|
||||
"DeleteBucketTagging_success": DeleteBucketTagging_success,
|
||||
"PutObject_non_existing_bucket": PutObject_non_existing_bucket,
|
||||
"PutObject_special_chars": PutObject_special_chars,
|
||||
"PutObject_invalid_long_tags": PutObject_invalid_long_tags,
|
||||
"PutObject_success": PutObject_success,
|
||||
"HeadObject_non_existing_object": HeadObject_non_existing_object,
|
||||
"HeadObject_success": HeadObject_success,
|
||||
"GetObject_non_existing_key": GetObject_non_existing_key,
|
||||
"GetObject_invalid_ranges": GetObject_invalid_ranges,
|
||||
"GetObject_with_meta": GetObject_with_meta,
|
||||
"GetObject_success": GetObject_success,
|
||||
"GetObject_by_range_success": GetObject_by_range_success,
|
||||
"ListObjects_non_existing_bucket": ListObjects_non_existing_bucket,
|
||||
"ListObjects_with_prefix": ListObjects_with_prefix,
|
||||
"ListObject_truncated": ListObject_truncated,
|
||||
"ListObjects_invalid_max_keys": ListObjects_invalid_max_keys,
|
||||
"ListObjects_max_keys_0": ListObjects_max_keys_0,
|
||||
"ListObjects_delimiter": ListObjects_delimiter,
|
||||
"ListObjects_max_keys_none": ListObjects_max_keys_none,
|
||||
"ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list,
|
||||
"ListObjectsV2_start_after": ListObjectsV2_start_after,
|
||||
"ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token,
|
||||
"ListObjectsV2_start_after_not_in_list": ListObjectsV2_start_after_not_in_list,
|
||||
"ListObjectsV2_start_after_empty_result": ListObjectsV2_start_after_empty_result,
|
||||
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
|
||||
"DeleteObject_success": DeleteObject_success,
|
||||
"DeleteObject_success_status_code": DeleteObject_success_status_code,
|
||||
"DeleteObjects_empty_input": DeleteObjects_empty_input,
|
||||
"DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects,
|
||||
"DeleteObjects_success": DeleteObjects_success,
|
||||
"CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket,
|
||||
"CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket,
|
||||
"CopyObject_copy_to_itself": CopyObject_copy_to_itself,
|
||||
"CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata,
|
||||
"CopyObject_success": CopyObject_success,
|
||||
"PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object,
|
||||
"PutObjectTagging_long_tags": PutObjectTagging_long_tags,
|
||||
"PutObjectTagging_success": PutObjectTagging_success,
|
||||
"GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object,
|
||||
"GetObjectTagging_success": GetObjectTagging_success,
|
||||
"DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object,
|
||||
"DeleteObjectTagging_success_status": DeleteObjectTagging_success_status,
|
||||
"DeleteObjectTagging_success": DeleteObjectTagging_success,
|
||||
"CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket,
|
||||
"CreateMultipartUpload_success": CreateMultipartUpload_success,
|
||||
"UploadPart_non_existing_bucket": UploadPart_non_existing_bucket,
|
||||
"UploadPart_invalid_part_number": UploadPart_invalid_part_number,
|
||||
"UploadPart_non_existing_key": UploadPart_non_existing_key,
|
||||
"UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload,
|
||||
"UploadPart_success": UploadPart_success,
|
||||
"UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket,
|
||||
"UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId,
|
||||
"UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key,
|
||||
"UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number,
|
||||
"UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source,
|
||||
"UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket,
|
||||
"UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key,
|
||||
"UploadPartCopy_success": UploadPartCopy_success,
|
||||
"UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range,
|
||||
"UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size,
|
||||
"UploadPartCopy_by_range_success": UploadPartCopy_by_range_success,
|
||||
"ListParts_incorrect_uploadId": ListParts_incorrect_uploadId,
|
||||
"ListParts_incorrect_object_key": ListParts_incorrect_object_key,
|
||||
"ListParts_success": ListParts_success,
|
||||
"ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket,
|
||||
"ListMultipartUploads_empty_result": ListMultipartUploads_empty_result,
|
||||
"ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads,
|
||||
"ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads,
|
||||
"ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker,
|
||||
"ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker,
|
||||
"ListMultipartUploads_success": ListMultipartUploads_success,
|
||||
"AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket,
|
||||
"AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId,
|
||||
"AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key,
|
||||
"AbortMultipartUpload_success": AbortMultipartUpload_success,
|
||||
"AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code,
|
||||
"CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket,
|
||||
"CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number,
|
||||
"CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag,
|
||||
"CompleteMultipartUpload_success": CompleteMultipartUpload_success,
|
||||
"PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket,
|
||||
"PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp,
|
||||
"PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants,
|
||||
"PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants,
|
||||
"PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner,
|
||||
"PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied,
|
||||
"PutBucketAcl_success_grants": PutBucketAcl_success_grants,
|
||||
"PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl,
|
||||
"PutBucketAcl_success_acp": PutBucketAcl_success_acp,
|
||||
"GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket,
|
||||
"GetBucketAcl_access_denied": GetBucketAcl_access_denied,
|
||||
"GetBucketAcl_success": GetBucketAcl_success,
|
||||
"PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj,
|
||||
"PutObject_overwrite_file_obj": PutObject_overwrite_file_obj,
|
||||
"PutObject_dir_obj_with_data": PutObject_dir_obj_with_data,
|
||||
"CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj,
|
||||
}
|
||||
}
|
||||
|
||||
1361
integration/tests.go
1361
integration/tests.go
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,7 @@ import (
|
||||
"io"
|
||||
rnd "math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
@@ -150,6 +151,20 @@ func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient) error) error {
|
||||
runF(testName)
|
||||
clt := s3.NewPresignClient(s3.NewFromConfig(s.Config()))
|
||||
|
||||
err := handler(clt)
|
||||
if err != nil {
|
||||
failF("%v: %v", testName, err)
|
||||
return fmt.Errorf("%v: %w", testName, err)
|
||||
}
|
||||
|
||||
passF(testName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
|
||||
req, err := http.NewRequest(method, fmt.Sprintf("%v/%v", endpoint, path), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
@@ -551,3 +566,26 @@ func genRandString(length int) string {
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
const (
|
||||
credAccess int = iota
|
||||
credDate
|
||||
credRegion
|
||||
credService
|
||||
credTerminator
|
||||
)
|
||||
|
||||
func changeAuthCred(uri, newVal string, index int) (string, error) {
|
||||
urlParsed, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
queries := urlParsed.Query()
|
||||
creds := strings.Split(queries.Get("X-Amz-Credential"), "/")
|
||||
creds[index] = newVal
|
||||
queries.Set("X-Amz-Credential", strings.Join(creds, "/"))
|
||||
urlParsed.RawQuery = queries.Encode()
|
||||
|
||||
return urlParsed.String(), nil
|
||||
}
|
||||
|
||||
@@ -44,6 +44,9 @@ var _ backend.Backend = &BackendMock{}
|
||||
// DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
|
||||
// panic("mock out the DeleteBucket method")
|
||||
// },
|
||||
// DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
|
||||
// panic("mock out the DeleteBucketTagging method")
|
||||
// },
|
||||
// DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
|
||||
// panic("mock out the DeleteObject method")
|
||||
// },
|
||||
@@ -56,6 +59,9 @@ var _ backend.Backend = &BackendMock{}
|
||||
// GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
|
||||
// panic("mock out the GetBucketAcl method")
|
||||
// },
|
||||
// GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
|
||||
// panic("mock out the GetBucketTagging method")
|
||||
// },
|
||||
// GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
|
||||
// panic("mock out the GetObject method")
|
||||
// },
|
||||
@@ -95,6 +101,9 @@ var _ backend.Backend = &BackendMock{}
|
||||
// PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
|
||||
// panic("mock out the PutBucketAcl method")
|
||||
// },
|
||||
// PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
|
||||
// panic("mock out the PutBucketTagging method")
|
||||
// },
|
||||
// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
|
||||
// panic("mock out the PutObject method")
|
||||
// },
|
||||
@@ -150,6 +159,9 @@ type BackendMock struct {
|
||||
// DeleteBucketFunc mocks the DeleteBucket method.
|
||||
DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
|
||||
|
||||
// DeleteBucketTaggingFunc mocks the DeleteBucketTagging method.
|
||||
DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
|
||||
|
||||
// DeleteObjectFunc mocks the DeleteObject method.
|
||||
DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error
|
||||
|
||||
@@ -162,6 +174,9 @@ type BackendMock struct {
|
||||
// GetBucketAclFunc mocks the GetBucketAcl method.
|
||||
GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
|
||||
|
||||
// GetBucketTaggingFunc mocks the GetBucketTagging method.
|
||||
GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
|
||||
|
||||
// GetObjectFunc mocks the GetObject method.
|
||||
GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error)
|
||||
|
||||
@@ -201,6 +216,9 @@ type BackendMock struct {
|
||||
// PutBucketAclFunc mocks the PutBucketAcl method.
|
||||
PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
|
||||
|
||||
// PutBucketTaggingFunc mocks the PutBucketTagging method.
|
||||
PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error
|
||||
|
||||
// PutObjectFunc mocks the PutObject method.
|
||||
PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error)
|
||||
|
||||
@@ -283,6 +301,13 @@ type BackendMock struct {
|
||||
// DeleteBucketInput is the deleteBucketInput argument value.
|
||||
DeleteBucketInput *s3.DeleteBucketInput
|
||||
}
|
||||
// DeleteBucketTagging holds details about calls to the DeleteBucketTagging method.
|
||||
DeleteBucketTagging []struct {
|
||||
// ContextMoqParam is the contextMoqParam argument value.
|
||||
ContextMoqParam context.Context
|
||||
// Bucket is the bucket argument value.
|
||||
Bucket string
|
||||
}
|
||||
// DeleteObject holds details about calls to the DeleteObject method.
|
||||
DeleteObject []struct {
|
||||
// ContextMoqParam is the contextMoqParam argument value.
|
||||
@@ -313,6 +338,13 @@ type BackendMock struct {
|
||||
// GetBucketAclInput is the getBucketAclInput argument value.
|
||||
GetBucketAclInput *s3.GetBucketAclInput
|
||||
}
|
||||
// GetBucketTagging holds details about calls to the GetBucketTagging method.
|
||||
GetBucketTagging []struct {
|
||||
// ContextMoqParam is the contextMoqParam argument value.
|
||||
ContextMoqParam context.Context
|
||||
// Bucket is the bucket argument value.
|
||||
Bucket string
|
||||
}
|
||||
// GetObject holds details about calls to the GetObject method.
|
||||
GetObject []struct {
|
||||
// ContextMoqParam is the contextMoqParam argument value.
|
||||
@@ -410,6 +442,15 @@ type BackendMock struct {
|
||||
// Data is the data argument value.
|
||||
Data []byte
|
||||
}
|
||||
// PutBucketTagging holds details about calls to the PutBucketTagging method.
|
||||
PutBucketTagging []struct {
|
||||
// ContextMoqParam is the contextMoqParam argument value.
|
||||
ContextMoqParam context.Context
|
||||
// Bucket is the bucket argument value.
|
||||
Bucket string
|
||||
// Tags is the tags argument value.
|
||||
Tags map[string]string
|
||||
}
|
||||
// PutObject holds details about calls to the PutObject method.
|
||||
PutObject []struct {
|
||||
// ContextMoqParam is the contextMoqParam argument value.
|
||||
@@ -477,10 +518,12 @@ type BackendMock struct {
|
||||
lockCreateBucket sync.RWMutex
|
||||
lockCreateMultipartUpload sync.RWMutex
|
||||
lockDeleteBucket sync.RWMutex
|
||||
lockDeleteBucketTagging sync.RWMutex
|
||||
lockDeleteObject sync.RWMutex
|
||||
lockDeleteObjectTagging sync.RWMutex
|
||||
lockDeleteObjects sync.RWMutex
|
||||
lockGetBucketAcl sync.RWMutex
|
||||
lockGetBucketTagging sync.RWMutex
|
||||
lockGetObject sync.RWMutex
|
||||
lockGetObjectAcl sync.RWMutex
|
||||
lockGetObjectAttributes sync.RWMutex
|
||||
@@ -494,6 +537,7 @@ type BackendMock struct {
|
||||
lockListObjectsV2 sync.RWMutex
|
||||
lockListParts sync.RWMutex
|
||||
lockPutBucketAcl sync.RWMutex
|
||||
lockPutBucketTagging sync.RWMutex
|
||||
lockPutObject sync.RWMutex
|
||||
lockPutObjectAcl sync.RWMutex
|
||||
lockPutObjectTagging sync.RWMutex
|
||||
@@ -765,6 +809,42 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
|
||||
return calls
|
||||
}
|
||||
|
||||
// DeleteBucketTagging calls DeleteBucketTaggingFunc.
|
||||
func (mock *BackendMock) DeleteBucketTagging(contextMoqParam context.Context, bucket string) error {
|
||||
if mock.DeleteBucketTaggingFunc == nil {
|
||||
panic("BackendMock.DeleteBucketTaggingFunc: method is nil but Backend.DeleteBucketTagging was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
}{
|
||||
ContextMoqParam: contextMoqParam,
|
||||
Bucket: bucket,
|
||||
}
|
||||
mock.lockDeleteBucketTagging.Lock()
|
||||
mock.calls.DeleteBucketTagging = append(mock.calls.DeleteBucketTagging, callInfo)
|
||||
mock.lockDeleteBucketTagging.Unlock()
|
||||
return mock.DeleteBucketTaggingFunc(contextMoqParam, bucket)
|
||||
}
|
||||
|
||||
// DeleteBucketTaggingCalls gets all the calls that were made to DeleteBucketTagging.
|
||||
// Check the length with:
|
||||
//
|
||||
// len(mockedBackend.DeleteBucketTaggingCalls())
|
||||
func (mock *BackendMock) DeleteBucketTaggingCalls() []struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
} {
|
||||
var calls []struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
}
|
||||
mock.lockDeleteBucketTagging.RLock()
|
||||
calls = mock.calls.DeleteBucketTagging
|
||||
mock.lockDeleteBucketTagging.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
// DeleteObject calls DeleteObjectFunc.
|
||||
func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
|
||||
if mock.DeleteObjectFunc == nil {
|
||||
@@ -913,6 +993,42 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
|
||||
return calls
|
||||
}
|
||||
|
||||
// GetBucketTagging calls GetBucketTaggingFunc.
|
||||
func (mock *BackendMock) GetBucketTagging(contextMoqParam context.Context, bucket string) (map[string]string, error) {
|
||||
if mock.GetBucketTaggingFunc == nil {
|
||||
panic("BackendMock.GetBucketTaggingFunc: method is nil but Backend.GetBucketTagging was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
}{
|
||||
ContextMoqParam: contextMoqParam,
|
||||
Bucket: bucket,
|
||||
}
|
||||
mock.lockGetBucketTagging.Lock()
|
||||
mock.calls.GetBucketTagging = append(mock.calls.GetBucketTagging, callInfo)
|
||||
mock.lockGetBucketTagging.Unlock()
|
||||
return mock.GetBucketTaggingFunc(contextMoqParam, bucket)
|
||||
}
|
||||
|
||||
// GetBucketTaggingCalls gets all the calls that were made to GetBucketTagging.
|
||||
// Check the length with:
|
||||
//
|
||||
// len(mockedBackend.GetBucketTaggingCalls())
|
||||
func (mock *BackendMock) GetBucketTaggingCalls() []struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
} {
|
||||
var calls []struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
}
|
||||
mock.lockGetBucketTagging.RLock()
|
||||
calls = mock.calls.GetBucketTagging
|
||||
mock.lockGetBucketTagging.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
// GetObject calls GetObjectFunc.
|
||||
func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
|
||||
if mock.GetObjectFunc == nil {
|
||||
@@ -1393,6 +1509,46 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
|
||||
return calls
|
||||
}
|
||||
|
||||
// PutBucketTagging calls PutBucketTaggingFunc.
|
||||
func (mock *BackendMock) PutBucketTagging(contextMoqParam context.Context, bucket string, tags map[string]string) error {
|
||||
if mock.PutBucketTaggingFunc == nil {
|
||||
panic("BackendMock.PutBucketTaggingFunc: method is nil but Backend.PutBucketTagging was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
Tags map[string]string
|
||||
}{
|
||||
ContextMoqParam: contextMoqParam,
|
||||
Bucket: bucket,
|
||||
Tags: tags,
|
||||
}
|
||||
mock.lockPutBucketTagging.Lock()
|
||||
mock.calls.PutBucketTagging = append(mock.calls.PutBucketTagging, callInfo)
|
||||
mock.lockPutBucketTagging.Unlock()
|
||||
return mock.PutBucketTaggingFunc(contextMoqParam, bucket, tags)
|
||||
}
|
||||
|
||||
// PutBucketTaggingCalls gets all the calls that were made to PutBucketTagging.
|
||||
// Check the length with:
|
||||
//
|
||||
// len(mockedBackend.PutBucketTaggingCalls())
|
||||
func (mock *BackendMock) PutBucketTaggingCalls() []struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
Tags map[string]string
|
||||
} {
|
||||
var calls []struct {
|
||||
ContextMoqParam context.Context
|
||||
Bucket string
|
||||
Tags map[string]string
|
||||
}
|
||||
mock.lockPutBucketTagging.RLock()
|
||||
calls = mock.calls.PutBucketTagging
|
||||
mock.lockPutBucketTagging.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
// PutObject calls PutObjectFunc.
|
||||
func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
|
||||
if mock.PutObjectFunc == nil {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -343,6 +343,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return &s3.ListObjectsOutput{}, nil
|
||||
},
|
||||
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
|
||||
return map[string]string{}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -365,6 +368,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
},
|
||||
},
|
||||
}
|
||||
appError := fiber.New()
|
||||
@@ -384,6 +390,24 @@ func TestS3ApiController_ListActions(t *testing.T) {
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Get-bucket-tagging-non-existing-bucket",
|
||||
app: appError,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket?tagging", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 404,
|
||||
},
|
||||
{
|
||||
name: "Get-bucket-tagging-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodGet, "/my-bucket?tagging", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Get-bucket-acl-success",
|
||||
app: app,
|
||||
@@ -492,6 +516,17 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
</AccessControlPolicy>
|
||||
`
|
||||
|
||||
tagBody := `
|
||||
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<TagSet>
|
||||
<Tag>
|
||||
<Key>organization</Key>
|
||||
<Value>marketing</Value>
|
||||
</Tag>
|
||||
</TagSet>
|
||||
</Tagging>
|
||||
`
|
||||
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
@@ -503,6 +538,9 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
CreateBucketFunc: func(context.Context, *s3.CreateBucketInput, []byte) error {
|
||||
return nil
|
||||
},
|
||||
PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
// Mock ctx.Locals
|
||||
@@ -543,6 +581,24 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Put-bucket-tagging-invalid-body",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 400,
|
||||
},
|
||||
{
|
||||
name: "Put-bucket-tagging-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Put-bucket-acl-invalid-acl",
|
||||
app: app,
|
||||
@@ -869,12 +925,12 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
|
||||
app := fiber.New()
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
return acldata, nil
|
||||
},
|
||||
DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
|
||||
return nil
|
||||
},
|
||||
DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -904,6 +960,15 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
},
|
||||
{
|
||||
name: "Delete-bucket-tagging-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?tagging", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 204,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
|
||||
@@ -38,7 +38,7 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
|
||||
if ctx.Method() == http.MethodPatch {
|
||||
return ctx.Next()
|
||||
}
|
||||
if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
|
||||
if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") && !ctx.Request().URI().QueryArgs().Has("tagging") {
|
||||
if err := auth.IsAdmin(acct, isRoot); err != nil {
|
||||
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
|
||||
}
|
||||
|
||||
@@ -44,6 +44,12 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
// If account is set in context locals, it means it was presigned url case
|
||||
_, ok := ctx.Locals("account").(auth.Account)
|
||||
if ok {
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
ctx.Locals("region", region)
|
||||
ctx.Locals("startTime", time.Now())
|
||||
authorization := ctx.Get("Authorization")
|
||||
@@ -96,7 +102,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
}
|
||||
|
||||
// Validate the dates difference
|
||||
err = validateDate(tdate)
|
||||
err = utils.ValidateDate(tdate)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
@@ -158,29 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
|
||||
return a.iam.GetUserAccount(access)
|
||||
}
|
||||
|
||||
func validateDate(date time.Time) error {
|
||||
now := time.Now().UTC()
|
||||
diff := date.Unix() - now.Unix()
|
||||
|
||||
// Checks the dates difference to be less than a minute
|
||||
if diff > 60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
if diff < -60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
69
s3api/middlewares/presign-auth.go
Normal file
69
s3api/middlewares/presign-auth.go
Normal file
@@ -0,0 +1,69 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
if ctx.Query("X-Amz-Signature") == "" {
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
ctx.Locals("region", region)
|
||||
ctx.Locals("startTime", time.Now())
|
||||
|
||||
authData, err := utils.ParsePresignedURIParts(ctx)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug)
|
||||
})
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
|
||||
app.Use(middlewares.RequestLogger(server.debug))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
|
||||
app.Use(middlewares.VerifyMD5Body(l))
|
||||
|
||||
@@ -126,16 +126,19 @@ func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string,
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
|
||||
AccessKeyID: auth.Access,
|
||||
SecretAccessKey: secret,
|
||||
}, req, checksum, service, auth.Region, tdate, func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
signErr := signer.SignHTTP(req.Context(),
|
||||
aws.Credentials{
|
||||
AccessKeyID: auth.Access,
|
||||
SecretAccessKey: secret,
|
||||
},
|
||||
req, checksum, service, auth.Region, tdate,
|
||||
func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("sign generated http request: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/valyala/fasthttp/fasthttputil"
|
||||
)
|
||||
|
||||
func TestAuthParse(t *testing.T) {
|
||||
@@ -10,18 +17,26 @@ func TestAuthParse(t *testing.T) {
|
||||
authstr string // Authorization string
|
||||
algo string
|
||||
sig string
|
||||
}{{
|
||||
name: "restic",
|
||||
authstr: "AWS4-HMAC-SHA256 Credential=user/20240116/us-east-1/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
|
||||
algo: "AWS4-HMAC-SHA256",
|
||||
sig: "d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
|
||||
},
|
||||
}{
|
||||
{
|
||||
name: "restic",
|
||||
authstr: "AWS4-HMAC-SHA256 Credential=user/20240116/us-east-1/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
|
||||
algo: "AWS4-HMAC-SHA256",
|
||||
sig: "d5199fc7f3aa35dd3d400427be2ae4c98bfad390785280cbb9eea015b51e12ac",
|
||||
},
|
||||
{
|
||||
name: "aws eaxample",
|
||||
authstr: "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-date, Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
|
||||
algo: "AWS4-HMAC-SHA256",
|
||||
sig: "fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
|
||||
}}
|
||||
},
|
||||
{
|
||||
name: "s3browser",
|
||||
authstr: "AWS4-HMAC-SHA256 Credential=access_key/20240206/us-east-1/s3/aws4_request,SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, Signature=37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6",
|
||||
algo: "AWS4-HMAC-SHA256",
|
||||
sig: "37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6",
|
||||
},
|
||||
}
|
||||
|
||||
for _, v := range vectors {
|
||||
t.Run(v.name, func(t *testing.T) {
|
||||
@@ -38,3 +53,78 @@ func TestAuthParse(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 2024/02/06 21:03:28 Request headers:
|
||||
// 2024/02/06 21:03:28 Host: 172.21.0.160:11000
|
||||
// 2024/02/06 21:03:28 User-Agent: S3 Browser/11.5.7 (https://s3browser.com)
|
||||
// 2024/02/06 21:03:28 Authorization: AWS4-HMAC-SHA256 Credential=access_key/20240206/us-east-1/s3/aws4_request,SignedHeaders=host;user-agent;x-amz-content-sha256;x-amz-date, Signature=37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6
|
||||
// 2024/02/06 21:03:28 X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
// 2024/02/06 21:03:28 X-Amz-Date: 20240206T210328Z
|
||||
func Test_Client_UserAgent(t *testing.T) {
|
||||
signedHdrs := []string{"host", "user-agent", "x-amz-content-sha256", "x-amz-date"}
|
||||
access := "access_key"
|
||||
secret := "secret_key"
|
||||
region := "us-east-1"
|
||||
host := "172.21.0.160:11000"
|
||||
agent := "S3 Browser/11.5.7 (https://s3browser.com)"
|
||||
expectedSig := "37a35d96998d786113ad420c57c22c5433f6aca74f88f26566caa047fc3601c6"
|
||||
dateStr := "20240206T210328Z"
|
||||
|
||||
app := fiber.New(fiber.Config{DisableStartupMessage: true})
|
||||
|
||||
tdate, err := time.Parse(iso8601Format, dateStr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
app.Get("/", func(c *fiber.Ctx) error {
|
||||
req, err := createHttpRequestFromCtx(c, signedHdrs, int64(c.Request().Header.ContentLength()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req.Host = host
|
||||
req.Header.Add("X-Amz-Content-Sha256", zeroLenSig)
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(),
|
||||
aws.Credentials{
|
||||
AccessKeyID: access,
|
||||
SecretAccessKey: secret,
|
||||
},
|
||||
req, zeroLenSig, service, region, tdate,
|
||||
func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
})
|
||||
if signErr != nil {
|
||||
t.Fatalf("sign generated http request: %v", err)
|
||||
}
|
||||
|
||||
genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if genAuth.Signature != expectedSig {
|
||||
t.Errorf("SIG: %v\nexpected: %v\n", genAuth.Signature, expectedSig)
|
||||
}
|
||||
|
||||
return c.Send(c.Request().Header.UserAgent())
|
||||
})
|
||||
|
||||
ln := fasthttputil.NewInmemoryListener()
|
||||
go func() {
|
||||
err := app.Listener(ln)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
c := fiber.AcquireClient()
|
||||
c.UserAgent = agent
|
||||
a := c.Get("http://example.com")
|
||||
a.HostClient.Dial = func(_ string) (net.Conn, error) { return ln.Dial() }
|
||||
a.String()
|
||||
fiber.ReleaseClient(c)
|
||||
}
|
||||
|
||||
243
s3api/utils/presign-auth-reader.go
Normal file
243
s3api/utils/presign-auth-reader.go
Normal file
@@ -0,0 +1,243 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/smithy-go/logging"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
const (
|
||||
unsignedPayload string = "UNSIGNED-PAYLOAD"
|
||||
)
|
||||
|
||||
// PresignedAuthReader is an io.Reader that validates presigned request authorization
|
||||
// once the underlying reader returns io.EOF. This is needed for streaming
|
||||
// data requests where the data size is not known until
|
||||
// the data is completely read.
|
||||
type PresignedAuthReader struct {
|
||||
ctx *fiber.Ctx
|
||||
auth AuthData
|
||||
secret string
|
||||
r io.Reader
|
||||
debug bool
|
||||
}
|
||||
|
||||
func NewPresignedAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *PresignedAuthReader {
|
||||
return &PresignedAuthReader{
|
||||
ctx: ctx,
|
||||
r: r,
|
||||
auth: auth,
|
||||
secret: secret,
|
||||
debug: debug,
|
||||
}
|
||||
}
|
||||
|
||||
// Read allows *PresignedAuthReader to be used as an io.Reader
|
||||
func (pr *PresignedAuthReader) Read(p []byte) (int, error) {
|
||||
n, err := pr.r.Read(p)
|
||||
|
||||
if errors.Is(err, io.EOF) {
|
||||
cerr := CheckPresignedSignature(pr.ctx, pr.auth, pr.secret, pr.debug)
|
||||
if cerr != nil {
|
||||
return n, cerr
|
||||
}
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// CheckPresignedSignature validates presigned request signature
|
||||
func CheckPresignedSignature(ctx *fiber.Ctx, auth AuthData, secret string, debug bool) error {
|
||||
signedHdrs := strings.Split(auth.SignedHeaders, ";")
|
||||
|
||||
var contentLength int64
|
||||
var err error
|
||||
contentLengthStr := ctx.Get("Content-Length")
|
||||
if contentLengthStr != "" {
|
||||
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new http request instance from fasthttp request
|
||||
req, err := createPresignedHttpRequestFromCtx(ctx, signedHdrs, contentLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create http request from context: %w", err)
|
||||
}
|
||||
|
||||
date, _ := time.Parse(iso8601Format, auth.Date)
|
||||
|
||||
signer := v4.NewSigner()
|
||||
uri, _, signErr := signer.PresignHTTP(ctx.Context(), aws.Credentials{
|
||||
AccessKeyID: auth.Access,
|
||||
SecretAccessKey: secret,
|
||||
}, req, unsignedPayload, service, auth.Region, date, func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("presign generated http request: %w", err)
|
||||
}
|
||||
|
||||
urlParts, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse presigned url: %w", err)
|
||||
}
|
||||
|
||||
signature := urlParts.Query().Get("X-Amz-Signature")
|
||||
if signature != auth.Signature {
|
||||
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
|
||||
//
|
||||
// # ParsePresignedURIParts parses and validates request URL query parameters
|
||||
//
|
||||
// ?X-Amz-Algorithm=AWS4-HMAC-SHA256
|
||||
// &X-Amz-Credential=access-key-id/20130721/us-east-1/s3/aws4_request
|
||||
// &X-Amz-Date=20130721T201207Z
|
||||
// &X-Amz-Expires=86400
|
||||
// &X-Amz-SignedHeaders=host
|
||||
// &X-Amz-Signature=1e68ad45c1db540284a4a1eca3884c293ba1a0ff63ab9db9a15b5b29dfa02cd8
|
||||
func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
|
||||
a := AuthData{}
|
||||
|
||||
// Get and verify algorithm query parameter
|
||||
algo := ctx.Query("X-Amz-Algorithm")
|
||||
if algo == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
if algo != "AWS4-HMAC-SHA256" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)
|
||||
}
|
||||
|
||||
// Parse and validate credentials query parameter
|
||||
credsQuery := ctx.Query("X-Amz-Credential")
|
||||
if credsQuery == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
creds := strings.Split(credsQuery, "/")
|
||||
if len(creds) != 5 {
|
||||
return a, s3err.GetAPIError(s3err.ErrCredMalformed)
|
||||
}
|
||||
if creds[3] != "s3" {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
|
||||
}
|
||||
if creds[4] != "aws4_request" {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
|
||||
}
|
||||
_, err := time.Parse(yyyymmdd, creds[1])
|
||||
if err != nil {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
|
||||
}
|
||||
|
||||
// Parse and validate Date query param
|
||||
date := ctx.Query("X-Amz-Date")
|
||||
if date == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
tdate, err := time.Parse(iso8601Format, date)
|
||||
if err != nil {
|
||||
return a, s3err.GetAPIError(s3err.ErrMalformedDate)
|
||||
}
|
||||
|
||||
if date[:8] != creds[1] {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
|
||||
}
|
||||
|
||||
if ctx.Locals("region") != creds[2] {
|
||||
return a, s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
|
||||
signature := ctx.Query("X-Amz-Signature")
|
||||
if signature == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
signedHdrs := ctx.Query("X-Amz-SignedHeaders")
|
||||
if signedHdrs == "" {
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
// Validate X-Amz-Expires query param and check if request is expired
|
||||
err = validateExpiration(ctx.Query("X-Amz-Expires"), tdate)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
|
||||
a.Signature = signature
|
||||
a.Access = creds[0]
|
||||
a.Algorithm = algo
|
||||
a.Region = creds[2]
|
||||
a.SignedHeaders = signedHdrs
|
||||
a.Date = date
|
||||
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func validateExpiration(str string, date time.Time) error {
|
||||
if str == "" {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
|
||||
exp, err := strconv.Atoi(str)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedExpires)
|
||||
}
|
||||
|
||||
if exp < 0 {
|
||||
return s3err.GetAPIError(s3err.ErrNegativeExpires)
|
||||
}
|
||||
|
||||
if exp > 604800 {
|
||||
return s3err.GetAPIError(s3err.ErrMaximumExpires)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
passed := int(now.Sub(date).Seconds())
|
||||
|
||||
if passed > exp {
|
||||
return s3err.GetAPIError(s3err.ErrExpiredPresignRequest)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
100
s3api/utils/presign-auth-reader_test.go
Normal file
100
s3api/utils/presign-auth-reader_test.go
Normal file
@@ -0,0 +1,100 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
func Test_validateExpiration(t *testing.T) {
|
||||
type args struct {
|
||||
str string
|
||||
date time.Time
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "empty-expiration",
|
||||
args: args{
|
||||
str: "",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrInvalidQueryParams),
|
||||
},
|
||||
{
|
||||
name: "invalid-expiration",
|
||||
args: args{
|
||||
str: "invalid_expiration",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrMalformedExpires),
|
||||
},
|
||||
{
|
||||
name: "negative-expiration",
|
||||
args: args{
|
||||
str: "-320",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrNegativeExpires),
|
||||
},
|
||||
{
|
||||
name: "exceeding-expiration",
|
||||
args: args{
|
||||
str: "6048000",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrMaximumExpires),
|
||||
},
|
||||
{
|
||||
name: "expired value",
|
||||
args: args{
|
||||
str: "200",
|
||||
date: time.Now().AddDate(0, 0, -1),
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrExpiredPresignRequest),
|
||||
},
|
||||
{
|
||||
name: "valid expiration",
|
||||
args: args{
|
||||
str: "300",
|
||||
date: time.Now(),
|
||||
},
|
||||
err: nil,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateExpiration(tt.args.str, tt.args.date)
|
||||
// Check for nil case
|
||||
if tt.err == nil && err != nil {
|
||||
t.Errorf("Expected nil error, got: %v", err)
|
||||
return
|
||||
} else if tt.err == nil && err == nil {
|
||||
// Both are nil, no need for further comparison
|
||||
return
|
||||
}
|
||||
|
||||
if err.Error() != tt.err.Error() {
|
||||
t.Errorf("Expected error: %v, got: %v", tt.err, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
48
s3api/utils/sign_hack.go
Normal file
48
s3api/utils/sign_hack.go
Normal file
@@ -0,0 +1,48 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// This is a hack to replace the default IgnoredHeaders in the aws-sdk-go-v2
|
||||
// internal/v4 package. Some AWS applications
|
||||
// (e.g. AWS Java SDK v1, Athena JDBC driver, s3 browser) sign the requests
|
||||
// including the User-Agent header. The aws sdk doesn't allow directly
|
||||
// modifying the ignored header list. Below is a hack to replace this list
|
||||
// with our own.
|
||||
|
||||
type Rule interface {
|
||||
IsValid(value string) bool
|
||||
}
|
||||
type Rules []Rule
|
||||
|
||||
//go:linkname __ignoredHeaders github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4.IgnoredHeaders
|
||||
var __ignoredHeaders unsafe.Pointer
|
||||
|
||||
func init() {
|
||||
// Avoids "go.info.github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4.IgnoredHeaders:
|
||||
// relocation target go.info.github.com/xxx/xxx/xxx.Rules not defined"
|
||||
var ignoredHeaders = (*Rules)(unsafe.Pointer(&__ignoredHeaders))
|
||||
|
||||
// clear the map, and set just the ignored headers we want
|
||||
reflect.ValueOf((*ignoredHeaders)[0]).FieldByName("Rule").Elem().Clear()
|
||||
reflect.ValueOf((*ignoredHeaders)[0]).FieldByName("Rule").Elem().SetMapIndex(
|
||||
reflect.ValueOf("Authorization"), reflect.ValueOf(struct{}{}))
|
||||
reflect.ValueOf((*ignoredHeaders)[0]).FieldByName("Rule").Elem().SetMapIndex(
|
||||
reflect.ValueOf("Expect"), reflect.ValueOf(struct{}{}))
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/valyala/fasthttp"
|
||||
@@ -86,6 +87,66 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
|
||||
return httpReq, nil
|
||||
}
|
||||
|
||||
var (
|
||||
signedQueryArgs = map[string]bool{
|
||||
"X-Amz-Algorithm": true,
|
||||
"X-Amz-Credential": true,
|
||||
"X-Amz-Date": true,
|
||||
"X-Amz-SignedHeaders": true,
|
||||
"X-Amz-Signature": true,
|
||||
}
|
||||
)
|
||||
|
||||
func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
|
||||
req := ctx.Request()
|
||||
var body io.Reader
|
||||
if IsBigDataAction(ctx) {
|
||||
body = req.BodyStream()
|
||||
} else {
|
||||
body = bytes.NewReader(req.Body())
|
||||
}
|
||||
|
||||
uri := string(ctx.Request().URI().Path())
|
||||
isFirst := true
|
||||
|
||||
ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
|
||||
_, ok := signedQueryArgs[string(key)]
|
||||
if !ok {
|
||||
if isFirst {
|
||||
uri += fmt.Sprintf("?%s=%s", key, value)
|
||||
isFirst = false
|
||||
} else {
|
||||
uri += fmt.Sprintf("&%s=%s", key, value)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
httpReq, err := http.NewRequest(string(req.Header.Method()), uri, body)
|
||||
if err != nil {
|
||||
return nil, errors.New("error in creating an http request")
|
||||
}
|
||||
// Set the request headers
|
||||
req.Header.VisitAll(func(key, value []byte) {
|
||||
keyStr := string(key)
|
||||
if includeHeader(keyStr, signedHdrs) {
|
||||
httpReq.Header.Add(keyStr, string(value))
|
||||
}
|
||||
})
|
||||
|
||||
// Check if Content-Length in signed headers
|
||||
// If content length is non 0, then the header will be included
|
||||
if !includeHeader("Content-Length", signedHdrs) {
|
||||
httpReq.ContentLength = 0
|
||||
} else {
|
||||
httpReq.ContentLength = contentLength
|
||||
}
|
||||
|
||||
// Set the Host header
|
||||
httpReq.Host = string(req.Header.Host())
|
||||
|
||||
return httpReq, nil
|
||||
}
|
||||
|
||||
func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
|
||||
ctx.Response().Header.DisableNormalizing()
|
||||
for key, val := range meta {
|
||||
@@ -149,3 +210,26 @@ func IsBigDataAction(ctx *fiber.Ctx) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ValidateDate(date time.Time) error {
|
||||
now := time.Now().UTC()
|
||||
diff := date.Unix() - now.Unix()
|
||||
|
||||
// Checks the dates difference to be less than a minute
|
||||
if diff > 60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
if diff < -60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
https://doc.s3.amazonaws.com/2006-03-01/AmazonS3.xsd
|
||||
|
||||
see https://blog.aqwari.net/xml-schema-go/
|
||||
|
||||
go install aqwari.net/xml/cmd/xsdgen@latest
|
||||
xsdgen -o s3api_xsd_generated.go -pkg s3response AmazonS3.xsd
|
||||
File diff suppressed because it is too large
Load Diff
@@ -16,6 +16,7 @@ package s3response
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
@@ -107,7 +108,8 @@ type TagSet struct {
|
||||
}
|
||||
|
||||
type Tagging struct {
|
||||
TagSet TagSet `xml:"TagSet"`
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging" json:"-"`
|
||||
TagSet TagSet `xml:"TagSet"`
|
||||
}
|
||||
|
||||
type DeleteObjects struct {
|
||||
@@ -139,3 +141,58 @@ type Bucket struct {
|
||||
Name string `json:"name"`
|
||||
Owner string `json:"owner"`
|
||||
}
|
||||
|
||||
type ListAllMyBucketsResult struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
|
||||
Owner CanonicalUser
|
||||
Buckets ListAllMyBucketsList
|
||||
}
|
||||
|
||||
type ListAllMyBucketsEntry struct {
|
||||
Name string
|
||||
CreationDate time.Time
|
||||
}
|
||||
|
||||
type ListAllMyBucketsList struct {
|
||||
Bucket []ListAllMyBucketsEntry
|
||||
}
|
||||
|
||||
type CanonicalUser struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
type CopyObjectResult struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||
LastModified time.Time
|
||||
ETag string
|
||||
}
|
||||
|
||||
type AccessControlPolicy struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
|
||||
Owner CanonicalUser
|
||||
AccessControlList AccessControlList
|
||||
}
|
||||
|
||||
type AccessControlList struct {
|
||||
Grant []Grant
|
||||
}
|
||||
|
||||
type Grant struct {
|
||||
Grantee Grantee
|
||||
Permission string
|
||||
}
|
||||
|
||||
// Set the following to encode correctly:
|
||||
//
|
||||
// Grantee: s3response.Grantee{
|
||||
// Xsi: "http://www.w3.org/2001/XMLSchema-instance",
|
||||
// Type: "CanonicalUser",
|
||||
// },
|
||||
type Grantee struct {
|
||||
XMLName xml.Name `xml:"Grantee"`
|
||||
Xsi string `xml:"xmlns:xsi,attr,omitempty"`
|
||||
Type string `xml:"xsi:type,attr,omitempty"`
|
||||
ID string
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
AWS_REGION=us-west-2
|
||||
AWS_REGION=us-east-1
|
||||
AWS_PROFILE=versity
|
||||
VERSITY_EXE=./versitygw
|
||||
BACKEND=posix
|
||||
LOCAL_FOLDER=/tmp/gw
|
||||
AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
BUCKET_ONE_NAME=versity-gwtest-bucket-one
|
||||
BUCKET_TWO_NAME=versity-gwtest-bucket-two
|
||||
RECREATE_BUCKETS=true
|
||||
@@ -3,4 +3,7 @@ AWS_PROFILE=versity
|
||||
VERSITY_EXE=./versitygw
|
||||
BACKEND=posix
|
||||
LOCAL_FOLDER=/tmp/gw
|
||||
AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
AWS_ENDPOINT_URL=http://127.0.0.1:7070
|
||||
BUCKET_ONE_NAME=versity-gwtest-bucket-one
|
||||
BUCKET_TWO_NAME=versity-gwtest-bucket-two
|
||||
RECREATE_BUCKETS=true
|
||||
197
tests/posix_tests.sh
Normal file → Executable file
197
tests/posix_tests.sh
Normal file → Executable file
@@ -1,139 +1,88 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
source ./tests/tests.sh
|
||||
|
||||
# check if object exists both on S3 and locally
|
||||
# param: object path
|
||||
# 0 for yes, 1 for no, 2 for error
|
||||
object_exists_remote_and_local() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "object existence check requires single name parameter"
|
||||
return 2
|
||||
fi
|
||||
object_exists "$1" || local exist_result=$?
|
||||
if [[ $exist_result -eq 2 ]]; then
|
||||
echo "Error checking if object exists"
|
||||
return 2
|
||||
fi
|
||||
if [[ $exist_result -eq 1 ]]; then
|
||||
echo "Error: object doesn't exist remotely"
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then
|
||||
echo "Error: object doesn't exist locally"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if object doesn't exist both on S3 and locally
|
||||
# param: object path
|
||||
# return 0 for doesn't exist, 1 for still exists, 2 for error
|
||||
object_not_exists_remote_and_local() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "object non-existence check requires single name parameter"
|
||||
return 2
|
||||
fi
|
||||
object_exists "$1" || local exist_result=$?
|
||||
if [[ $exist_result -eq 2 ]]; then
|
||||
echo "Error checking if object doesn't exist"
|
||||
return 2
|
||||
fi
|
||||
if [[ $exist_result -eq 0 ]]; then
|
||||
echo "Error: object exists remotely"
|
||||
return 1
|
||||
fi
|
||||
if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then
|
||||
echo "Error: object exists locally"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if a bucket doesn't exist both on S3 and on gateway
|
||||
# param: bucket name
|
||||
# return: 0 for doesn't exist, 1 for does, 2 for error
|
||||
bucket_not_exists_remote_and_local() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket existence check requires single name parameter"
|
||||
return 2
|
||||
fi
|
||||
bucket_exists "$1" || local exist_result=$?
|
||||
if [[ $exist_result -eq 2 ]]; then
|
||||
echo "Error checking if bucket exists"
|
||||
return 2
|
||||
fi
|
||||
if [[ $exist_result -eq 0 ]]; then
|
||||
echo "Error: bucket exists remotely"
|
||||
return 1
|
||||
fi
|
||||
if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then
|
||||
echo "Error: bucket exists locally"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if a bucket exists both on S3 and on gateway
|
||||
# param: bucket name
|
||||
# return: 0 for yes, 1 for no, 2 for error
|
||||
bucket_exists_remote_and_local() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket existence check requires single name parameter"
|
||||
return 2
|
||||
fi
|
||||
bucket_exists "$1" || local exist_result=$?
|
||||
if [[ $exist_result -eq 2 ]]; then
|
||||
echo "Error checking if bucket exists"
|
||||
return 2
|
||||
fi
|
||||
if [[ $exist_result -eq 1 ]]; then
|
||||
echo "Error: bucket doesn't exist remotely"
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then
|
||||
echo "Error: bucket doesn't exist locally"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
source ./tests/setup.sh
|
||||
source ./tests/util.sh
|
||||
source ./tests/util_posix.sh
|
||||
|
||||
# test that changes to local folders and files are reflected on S3
|
||||
@test test_local_creation_deletion {
|
||||
@test "test_local_creation_deletion" {
|
||||
|
||||
if [[ $RECREATE_BUCKETS != "true" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
local bucket_name="versity-gwtest-put-object-test"
|
||||
local object_name="test-object"
|
||||
|
||||
bucket_exists_remote_and_local $bucket_name || local bucket_exists=$?
|
||||
if [[ $bucket_exists -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
local object="$bucket_name"/"$object_name"
|
||||
if [[ $bucket_exists -eq 0 ]]; then
|
||||
object_exists_remote_and_local "$object" || local object_exists=$?
|
||||
if [[ $object_exists -eq 2 ]]; then
|
||||
fail "Object existence check error"
|
||||
fi
|
||||
if [[ $object_exists -eq 0 ]]; then
|
||||
delete_object "$object" || local delete_object=$?
|
||||
[[ $delete_object -eq 0 ]] || fail "Failed to delete object"
|
||||
fi
|
||||
delete_bucket $bucket_name || local delete_bucket=$?
|
||||
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
|
||||
fi
|
||||
mkdir "$LOCAL_FOLDER"/$bucket_name
|
||||
touch "$LOCAL_FOLDER"/$object
|
||||
bucket_exists_remote_and_local $bucket_name || local bucket_exists_two=$?
|
||||
mkdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
local object="$BUCKET_ONE_NAME"/"$object_name"
|
||||
touch "$LOCAL_FOLDER"/"$object"
|
||||
|
||||
bucket_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_exists_two=$?
|
||||
[[ $bucket_exists_two -eq 0 ]] || fail "Failed bucket existence check"
|
||||
object_exists_remote_and_local $object || local object_exists_two=$?
|
||||
object_exists_remote_and_local "$object" || local object_exists_two=$?
|
||||
[[ $object_exists_two -eq 0 ]] || fail "Failed object existence check"
|
||||
rm "$LOCAL_FOLDER"/$object
|
||||
|
||||
rm "$LOCAL_FOLDER"/"$object"
|
||||
sleep 1
|
||||
object_not_exists_remote_and_local $object || local object_deleted=$?
|
||||
object_not_exists_remote_and_local "$object" || local object_deleted=$?
|
||||
[[ $object_deleted -eq 0 ]] || fail "Failed object deletion check"
|
||||
rmdir "$LOCAL_FOLDER"/$bucket_name
|
||||
|
||||
rmdir "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
sleep 1
|
||||
bucket_not_exists_remote_and_local $bucket_name || local bucket_deleted=$?
|
||||
bucket_not_exists_remote_and_local "$BUCKET_ONE_NAME" || local bucket_deleted=$?
|
||||
[[ $bucket_deleted -eq 0 ]] || fail "Failed bucket deletion check"
|
||||
}
|
||||
|
||||
# test head-object command
|
||||
@test "test_head_object" {
|
||||
|
||||
local bucket_name=$BUCKET_ONE_NAME
|
||||
local object_name="object-one"
|
||||
|
||||
create_test_files $object_name
|
||||
if [ -e "$LOCAL_FOLDER"/"$bucket_name"/$object_name ]; then
|
||||
chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
|
||||
fi
|
||||
setup_bucket "$bucket_name" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating bucket"
|
||||
put_object "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name" || local result="$?"
|
||||
[[ result -eq 0 ]] || fail "Error adding object one"
|
||||
|
||||
chmod 000 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
|
||||
sleep 1
|
||||
object_is_accessible "$bucket_name" $object_name || local accessible=$?
|
||||
[[ $accessible -eq 1 ]] || fail "Object should be inaccessible"
|
||||
|
||||
chmod 755 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
|
||||
sleep 1
|
||||
object_is_accessible "$bucket_name" $object_name || local accessible_two=$?
|
||||
[[ $accessible_two -eq 0 ]] || fail "Object should be accessible"
|
||||
|
||||
delete_object "$bucket_name"/$object_name
|
||||
delete_bucket_or_contents "$bucket_name"
|
||||
delete_test_files $object_name
|
||||
}
|
||||
|
||||
# check info, accessiblity of bucket
|
||||
@test "test_get_bucket_info" {
|
||||
|
||||
if [ -e "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME" ]; then
|
||||
chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
sleep 1
|
||||
else
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating bucket"
|
||||
fi
|
||||
|
||||
chmod 000 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
sleep 1
|
||||
bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible=$?
|
||||
[[ $accessible -eq 1 ]] || fail "Bucket should be inaccessible"
|
||||
|
||||
chmod 755 "$LOCAL_FOLDER"/"$BUCKET_ONE_NAME"
|
||||
sleep 1
|
||||
bucket_is_accessible "$BUCKET_ONE_NAME" || local accessible_two=$?
|
||||
[[ $accessible_two -eq 0 ]] || fail "Bucket should be accessible"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
}
|
||||
|
||||
@@ -1,339 +1,95 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
source ./tests/tests.sh
|
||||
|
||||
# create an AWS bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
create_bucket() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "create bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 mb s3://"$1" 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error creating bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# delete an AWS bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
delete_bucket() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rb s3://"$1" 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
if [[ "$error" == *"The specified bucket does not exist"* ]]; then
|
||||
return 0
|
||||
else
|
||||
echo "error deleting bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if bucket exists
|
||||
# param: bucket name
|
||||
# return 0 for true, 1 for false, 2 for error
|
||||
bucket_exists() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket exists check missing bucket name"
|
||||
return 2
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
|
||||
echo "Exit code: $exit_code, error: $error"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then
|
||||
return 1
|
||||
else
|
||||
echo "error checking if bucket exists: $error"
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# create bucket if it doesn't exist
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
check_and_create_bucket() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket creation function requires bucket name"
|
||||
return 1
|
||||
fi
|
||||
local exists_result
|
||||
bucket_exists "$1" || exists_result=$?
|
||||
if [[ $exists_result -eq 2 ]]; then
|
||||
echo "Bucket existence check error"
|
||||
return 1
|
||||
fi
|
||||
local create_result
|
||||
if [[ $exists_result -eq 1 ]]; then
|
||||
create_bucket "$1" || create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
echo "Error creating bucket"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if object exists on S3 via gateway
|
||||
# param: object path
|
||||
# return 0 for true, 1 for false, 2 for error
|
||||
object_exists() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "object exists check missing object name"
|
||||
return 2
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
if [[ "$error" == "" ]]; then
|
||||
return 1
|
||||
else
|
||||
echo "error checking if object exists: $error"
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# add object to versitygw
|
||||
# params: source file, destination copy location
|
||||
# return 0 for success, 1 for failure
|
||||
put_object() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "put object command requires source, destination"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 cp "$1" s3://"$2" 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error copying object to bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# add object to versitygw if it doesn't exist
|
||||
# params: source file, destination copy location
|
||||
# return 0 for success or already exists, 1 for failure
|
||||
check_and_put_object() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "check and put object function requires source, destination"
|
||||
return 1
|
||||
fi
|
||||
object_exists "$2" || local exists_result=$?
|
||||
if [ $exists_result -eq 2 ]; then
|
||||
echo "error checking if object exists"
|
||||
return 1
|
||||
fi
|
||||
if [ $exists_result -eq 1 ]; then
|
||||
put_object "$1" "$2" || local put_result=$?
|
||||
if [ $put_result -ne 0 ]; then
|
||||
echo "error adding object"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# delete object from versitygw
|
||||
# param: object location
|
||||
# return 0 for success, 1 for failure
|
||||
delete_object() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete object command requires object parameter"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rm s3://"$1" 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error deleting object: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# list buckets on versitygw
|
||||
# no params
|
||||
# export bucket_array (bucket names) on success, return 1 for failure
|
||||
list_buckets() {
|
||||
local exit_code=0
|
||||
local output
|
||||
output=$(aws s3 ls 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error listing buckets: $output"
|
||||
return 1
|
||||
fi
|
||||
|
||||
bucket_array=()
|
||||
while IFS= read -r line; do
|
||||
bucket_name=$(echo "$line" | awk '{print $NF}')
|
||||
bucket_array+=("$bucket_name")
|
||||
done <<< "$output"
|
||||
|
||||
export bucket_array
|
||||
}
|
||||
|
||||
# list objects on versitygw, in bucket or folder
|
||||
# param: path of bucket or folder
|
||||
# export object_array (object names) on success, return 1 for failure
|
||||
list_objects() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "list objects command requires bucket or folder"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
local output
|
||||
output=$(aws s3 ls s3://"$1" 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error listing objects: $output"
|
||||
return 1
|
||||
fi
|
||||
|
||||
object_array=()
|
||||
while IFS= read -r line; do
|
||||
object_name=$(echo "$line" | awk '{print $NF}')
|
||||
object_array+=("$object_name")
|
||||
done <<< "$output"
|
||||
|
||||
export object_array
|
||||
}
|
||||
source ./tests/setup.sh
|
||||
source ./tests/util.sh
|
||||
|
||||
# test creation and deletion of bucket on versitygw
|
||||
@test "create_delete_bucket_test" {
|
||||
@test "test_create_delete_bucket" {
|
||||
|
||||
local bucket_name="versity-gwtest-create-delete-bucket-test"
|
||||
if [[ $RECREATE_BUCKETS != "true" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
bucket_exists $bucket_name || local exists=$?
|
||||
if [[ $exists -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
if [[ $exists -eq 0 ]]; then
|
||||
delete_bucket $bucket_name || local delete_result=$?
|
||||
[[ $delete_result -eq 0 ]] || fail "Failed to delete bucket"
|
||||
bucket_exists $bucket_name || local exists_two=$?
|
||||
[[ $exists_two -eq 1 ]] || fail "Failed bucket deletion"
|
||||
fi
|
||||
create_bucket $bucket_name || local create_result=$?
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local create_result=$?
|
||||
[[ $create_result -eq 0 ]] || fail "Failed to create bucket"
|
||||
bucket_exists $bucket_name || local exists_three=$?
|
||||
|
||||
bucket_exists "$BUCKET_ONE_NAME" || local exists_three=$?
|
||||
[[ $exists_three -eq 0 ]] || fail "Failed bucket existence check"
|
||||
delete_bucket $bucket_name || local delete_result_two=$?
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME" || local delete_result_two=$?
|
||||
[[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
|
||||
}
|
||||
|
||||
# test adding and removing an object on versitygw
|
||||
@test "put_object_test" {
|
||||
@test "test_put_object" {
|
||||
|
||||
local bucket_name="versity-gwtest-put-object-test"
|
||||
local object_name="test-object"
|
||||
|
||||
bucket_exists $bucket_name || local bucket_exists=$?
|
||||
if [[ $bucket_exists -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
local object="$bucket_name"/"$object_name"
|
||||
if [[ $bucket_exists -eq 0 ]]; then
|
||||
object_exists "$object" || local object_exists=$?
|
||||
if [[ $object_exists -eq 2 ]]; then
|
||||
fail "Object existence check error"
|
||||
fi
|
||||
if [[ $object_exists -eq 0 ]]; then
|
||||
delete_object "$object" || local delete_object=$?
|
||||
[[ $delete_object -eq 0 ]] || fail "Failed to delete object"
|
||||
fi
|
||||
delete_bucket $bucket_name || local delete_bucket=$?
|
||||
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
|
||||
fi
|
||||
touch "$object_name"
|
||||
create_bucket $bucket_name || local create_bucket=$?
|
||||
[[ $create_bucket -eq 0 ]] || fail "Failed to create bucket"
|
||||
put_object "$object_name" "$object" || local put_object=$?
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local setup_result=$?
|
||||
[[ $setup_result -eq 0 ]] || fail "error setting up bucket"
|
||||
|
||||
create_test_files "$object_name" || local create_result=$?
|
||||
|
||||
object="$BUCKET_ONE_NAME"/$object_name
|
||||
put_object "$test_file_folder"/"$object_name" "$object" || local put_object=$?
|
||||
[[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
|
||||
object_exists "$object" || local object_exists_two=$?
|
||||
[[ $object_exists_two -eq 0 ]] || fail "Object not added to bucket"
|
||||
delete_object "$object" || local delete_object_two=$?
|
||||
[[ $delete_object_two -eq 0 ]] || fail "Failed to delete object"
|
||||
delete_bucket $bucket_name || local delete_bucket=$?
|
||||
[[ $delete_bucket -eq 0 ]] || fail "Failed to delete bucket"
|
||||
rm "$object_name"
|
||||
object_exists "$object" || local exists_result_one=$?
|
||||
[[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
|
||||
|
||||
delete_object "$object" || local delete_result=$?
|
||||
[[ $delete_result -eq 0 ]] || fail "Failed to delete object"
|
||||
object_exists "$object" || local exists_result_two=$?
|
||||
[[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$object_name"
|
||||
}
|
||||
|
||||
# test listing buckets on versitygw
|
||||
@test "test_list_buckets" {
|
||||
|
||||
bucket_name_one="versity-gwtest-list-one"
|
||||
bucket_name_two="versity-gwtest-list-two"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local setup_result_one=$?
|
||||
[[ $setup_result_one -eq 0 ]] || fail "Bucket one setup error"
|
||||
setup_bucket "$BUCKET_TWO_NAME" || local setup_result_two=$?
|
||||
[[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error"
|
||||
|
||||
bucket_exists $bucket_name_one || local exists=$?
|
||||
if [[ $exists -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
if [[ $exists -eq 1 ]]; then
|
||||
create_bucket $bucket_name_one || local bucket_create_one=$?
|
||||
[[ $bucket_create_one -eq 0 ]] || fail "Failed to create bucket"
|
||||
fi
|
||||
bucket_exists $bucket_name_two || local exists_two=$?
|
||||
if [[ $exists_two -eq 2 ]]; then
|
||||
fail "Bucket existence check error"
|
||||
fi
|
||||
if [[ $exists_two -eq 1 ]]; then
|
||||
create_bucket $bucket_name_two || local bucket_create_two=$?
|
||||
[[ $bucket_create_two -eq 0 ]] || fail "Failed to create bucket"
|
||||
fi
|
||||
list_buckets
|
||||
local bucket_one_found=false
|
||||
local bucket_two_found=false
|
||||
for bucket in "${bucket_array[@]}"; do
|
||||
if [ "$bucket" == $bucket_name_one ]; then
|
||||
if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then
|
||||
bucket_one_found=true
|
||||
elif [ "$bucket" == $bucket_name_two ]; then
|
||||
elif [ "$bucket" == "$BUCKET_TWO_NAME" ]; then
|
||||
bucket_two_found=true
|
||||
fi
|
||||
if [ $bucket_one_found == true ] && [ $bucket_two_found == true ]; then
|
||||
return
|
||||
break
|
||||
fi
|
||||
done
|
||||
fail "$bucket_name_one and/or $bucket_name_two not listed (all buckets: ${bucket_array[*]})"
|
||||
delete_bucket $bucket_name_one || local deleted_one=$?
|
||||
[[ $deleted_one -eq 0 ]] || fail "Failed to delete bucket one"
|
||||
delete_bucket $bucket_name_two || local deleted_two=$?
|
||||
[[ $deleted_two -eq 0 ]] || fail "Failed to delete bucket one"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_bucket_or_contents "$BUCKET_TWO_NAME"
|
||||
|
||||
if [ $bucket_one_found != true ] || [ $bucket_two_found != true ]; then
|
||||
fail "'$BUCKET_ONE_NAME' and/or '$BUCKET_TWO_NAME' not listed (all buckets: ${bucket_array[*]})"
|
||||
fi
|
||||
}
|
||||
|
||||
# test listing a bucket's objects on versitygw
|
||||
@test test_list_objects {
|
||||
@test "test_list_objects" {
|
||||
|
||||
bucket_name="versity-gwtest-list-object"
|
||||
object_one="test-file-one"
|
||||
object_two="test-file-two"
|
||||
|
||||
touch $object_one $object_two
|
||||
check_and_create_bucket $bucket_name || local result_one=$?
|
||||
create_test_files $object_one $object_two
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result_one=$?
|
||||
[[ result_one -eq 0 ]] || fail "Error creating bucket"
|
||||
put_object $object_one "$bucket_name"/"$object_one" || local result_two=$?
|
||||
put_object "$test_file_folder"/$object_one "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$?
|
||||
[[ result_two -eq 0 ]] || fail "Error adding object one"
|
||||
put_object $object_two "$bucket_name"/"$object_two" || local result_three=$?
|
||||
put_object "$test_file_folder"/$object_two "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
|
||||
[[ result_three -eq 0 ]] || fail "Error adding object two"
|
||||
list_objects $bucket_name
|
||||
|
||||
list_objects "$BUCKET_ONE_NAME"
|
||||
local object_one_found=false
|
||||
local object_two_found=false
|
||||
for object in "${object_array[@]}"; do
|
||||
@@ -343,11 +99,305 @@ list_objects() {
|
||||
object_two_found=true
|
||||
fi
|
||||
done
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $object_one $object_two
|
||||
|
||||
if [ $object_one_found != true ] || [ $object_two_found != true ]; then
|
||||
fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
|
||||
fi
|
||||
delete_object "$bucket_name"/"$object_one"
|
||||
delete_object "$bucket_name"/"$object_two"
|
||||
delete_bucket $bucket_name
|
||||
rm $object_one $object_two
|
||||
}
|
||||
|
||||
# test ability to retrieve bucket ACLs
|
||||
@test "test_get_bucket_acl" {
|
||||
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating bucket"
|
||||
|
||||
get_bucket_acl "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Error retrieving acl"
|
||||
|
||||
id=$(echo "$acl" | jq '.Owner.ID')
|
||||
[[ $id == '"'"$AWS_ACCESS_KEY_ID"'"' ]] || fail "Acl mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
}
|
||||
|
||||
# test ability to delete multiple objects from bucket
|
||||
@test "test_delete_objects" {
|
||||
|
||||
local object_one="test-file-one"
|
||||
local object_two="test-file-two"
|
||||
|
||||
create_test_files "$object_one" "$object_two" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result_one=$?
|
||||
[[ $result_one -eq 0 ]] || fail "Error creating bucket"
|
||||
|
||||
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$?
|
||||
[[ $result_two -eq 0 ]] || fail "Error adding object one"
|
||||
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
|
||||
[[ $result_three -eq 0 ]] || fail "Error adding object two"
|
||||
|
||||
error=$(aws s3api delete-objects --bucket "$BUCKET_ONE_NAME" --delete '{
|
||||
"Objects": [
|
||||
{"Key": "test-file-one"},
|
||||
{"Key": "test-file-two"}
|
||||
]
|
||||
}') || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Error deleting objects: $error"
|
||||
|
||||
object_exists "$BUCKET_ONE_NAME"/"$object_one" || local exists_one=$?
|
||||
[[ $exists_one -eq 1 ]] || fail "Object one not deleted"
|
||||
object_exists "$BUCKET_ONE_NAME"/"$object_two" || local exists_two=$?
|
||||
[[ $exists_two -eq 1 ]] || fail "Object two not deleted"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$object_one" "$object_two"
|
||||
}
|
||||
|
||||
# test abilty to set and retrieve bucket tags
|
||||
@test "test-set-get-bucket-tags" {
|
||||
|
||||
local key="test_key"
|
||||
local value="test_value"
|
||||
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
get_bucket_tags "$BUCKET_ONE_NAME" || local get_result=$?
|
||||
[[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
|
||||
tag_set=$(echo "$tags" | jq '.TagSet')
|
||||
[[ $tag_set == "[]" ]] || fail "Error: tags not empty"
|
||||
|
||||
put_bucket_tag "$BUCKET_ONE_NAME" $key $value
|
||||
get_bucket_tags "$BUCKET_ONE_NAME" || local get_result_two=$?
|
||||
[[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
|
||||
tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
|
||||
tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
|
||||
[[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
|
||||
[[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
}
|
||||
|
||||
# test v1 s3api list objects command
|
||||
@test "test-s3api-list-objects-v1" {
|
||||
|
||||
local object_one="test-file-one"
|
||||
local object_two="test-file-two"
|
||||
local object_two_data="test data\n"
|
||||
|
||||
create_test_files "$object_one" "$object_two" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
|
||||
[[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
|
||||
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
|
||||
[[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
|
||||
|
||||
list_objects_s3api_v1 "$BUCKET_ONE_NAME"
|
||||
key_one=$(echo "$objects" | jq '.Contents[0].Key')
|
||||
[[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
|
||||
size_one=$(echo "$objects" | jq '.Contents[0].Size')
|
||||
[[ $size_one -eq 0 ]] || fail "Object one size mismatch"
|
||||
key_two=$(echo "$objects" | jq '.Contents[1].Key')
|
||||
[[ $key_two == '"'$object_two'"' ]] || fail "Object two mismatch"
|
||||
size_two=$(echo "$objects" | jq '.Contents[1].Size')
|
||||
[[ $size_two -eq ${#object_two_data} ]] || fail "Object two size mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$object_one" "$object_two"
|
||||
}
|
||||
|
||||
# test v2 s3api list objects command
|
||||
@test "test-s3api-list-objects-v2" {
|
||||
|
||||
local object_one="test-file-one"
|
||||
local object_two="test-file-two"
|
||||
local object_two_data="test data\n"
|
||||
|
||||
create_test_files "$object_one" "$object_two" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
|
||||
[[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
|
||||
put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
|
||||
[[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
|
||||
|
||||
list_objects_s3api_v2 "$BUCKET_ONE_NAME"
|
||||
key_one=$(echo "$objects" | jq '.Contents[0].Key')
|
||||
[[ $key_one == '"'$object_one'"' ]] || fail "Object one mismatch"
|
||||
size_one=$(echo "$objects" | jq '.Contents[0].Size')
|
||||
[[ $size_one -eq 0 ]] || fail "Object one size mismatch"
|
||||
key_two=$(echo "$objects" | jq '.Contents[1].Key')
|
||||
[[ $key_two == '"'$object_two'"' ]] || fail "Object two mismatch"
|
||||
size_two=$(echo "$objects" | jq '.Contents[1].Size')
|
||||
[[ $size_two -eq ${#object_two_data} ]] || fail "Object two size mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$object_one" "$object_two"
|
||||
}
|
||||
|
||||
# test abilty to set and retrieve object tags
|
||||
@test "test-set-get-object-tags" {
|
||||
|
||||
local bucket_file="bucket-file"
|
||||
local key="test_key"
|
||||
local value="test_value"
|
||||
|
||||
create_test_files "$bucket_file" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
|
||||
put_object "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
|
||||
[[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
|
||||
[[ $get_result -eq 0 ]] || fail "Error getting object tags"
|
||||
tag_set=$(echo "$tags" | jq '.TagSet')
|
||||
[[ $tag_set == "[]" ]] || fail "Error: tags not empty"
|
||||
|
||||
put_object_tag "$BUCKET_ONE_NAME" $bucket_file $key $value
|
||||
get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
|
||||
[[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
|
||||
tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
|
||||
tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
|
||||
[[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
|
||||
[[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $bucket_file
|
||||
}
|
||||
|
||||
# test multi-part upload
|
||||
@test "test-multi-part-upload" {
|
||||
|
||||
local bucket_file="bucket-file"
|
||||
bucket_file_data="test file\n"
|
||||
|
||||
create_test_files "$bucket_file" || local created=$?
|
||||
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$?
|
||||
[[ $upload_result -eq 0 ]] || fail "Error performing multipart upload"
|
||||
|
||||
copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy"
|
||||
copy_data=$(<"$test_file_folder"/$bucket_file-copy)
|
||||
[[ $bucket_file_data == "$copy_data" ]] || fail "Data doesn't match"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $bucket_file
|
||||
}
|
||||
|
||||
# test multi-part upload abort
|
||||
@test "test-multi-part-upload-abort" {
|
||||
|
||||
local bucket_file="bucket-file"
|
||||
bucket_file_data="test file\n"
|
||||
|
||||
create_test_files "$bucket_file" || local created=$?
|
||||
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$?
|
||||
[[ $abort_result -eq 0 ]] || fail "Abort failed"
|
||||
|
||||
object_exists "$BUCKET_ONE_NAME/$bucket_file" || exists=$?
|
||||
[[ $exists -eq 1 ]] || fail "Upload file exists after abort"
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $bucket_file
|
||||
}
|
||||
|
||||
# test multi-part upload list parts command
|
||||
@test "test-multipart-upload-list-parts" {
|
||||
|
||||
local bucket_file="bucket-file"
|
||||
local bucket_file_data="test file\n"
|
||||
|
||||
create_test_files "$bucket_file" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
list_parts "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || list_result=$?
|
||||
[[ list_result -eq 0 ]] || fail "Listing multipart upload parts failed"
|
||||
|
||||
declare -a parts_map
|
||||
for ((i=0;i<$4;i++)) {
|
||||
local part_number
|
||||
local etag
|
||||
part_number=$(echo "$parts" | jq ".[$i].PartNumber")
|
||||
if [[ $part_number -eq "" ]]; then
|
||||
echo "error: blank part number"
|
||||
return 1
|
||||
fi
|
||||
etag=$(echo "$parts" | jq ".[$i].ETag")
|
||||
if [[ $etag == "" ]]; then
|
||||
echo "error: blank etag"
|
||||
return 1
|
||||
fi
|
||||
parts_map[$part_number]=$etag
|
||||
}
|
||||
|
||||
for ((i=0;i<$4;i++)) {
|
||||
local part_number
|
||||
local etag
|
||||
part_number=$(echo "$listed_parts" | jq ".Parts[$i].PartNumber")
|
||||
etag=$(echo "$listed_parts" | jq ".Parts[$i].ETag")
|
||||
if [[ ${parts_map[$part_number]} != "$etag" ]]; then
|
||||
echo "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files $bucket_file
|
||||
}
|
||||
|
||||
# test listing of active uploads
|
||||
@test "test-multipart-upload-list-uploads" {
|
||||
|
||||
local bucket_file_one="bucket-file-one"
|
||||
local bucket_file_two="bucket-file-two"
|
||||
|
||||
create_test_files "$bucket_file_one" "$bucket_file_two" || local created=$?
|
||||
[[ $created -eq 0 ]] || fail "Error creating test files"
|
||||
setup_bucket "$BUCKET_ONE_NAME" || local result=$?
|
||||
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
|
||||
|
||||
list_multipart_uploads "$BUCKET_ONE_NAME" "$test_file_folder"/"$bucket_file_one" "$test_file_folder"/"$bucket_file_two"
|
||||
[[ $? -eq 0 ]] || fail "failed to list multipart uploads"
|
||||
|
||||
local key_one
|
||||
local key_two
|
||||
key_one=$(echo "$uploads" | jq '.Uploads[0].Key')
|
||||
key_two=$(echo "$uploads" | jq '.Uploads[1].Key')
|
||||
key_one=${key_one//\"/}
|
||||
key_two=${key_two//\"/}
|
||||
echo "$test_file_folder/${bucket_file_one}abc"
|
||||
echo "${key_one}abc"
|
||||
echo "Length of test_file_folder/bucket_file_one: ${#test_file_folder}/${#bucket_file_one}"
|
||||
echo "Length of key_one: ${#key_one}"
|
||||
if [[ "$test_file_folder/$bucket_file_one" != *"$key_one" ]]; then
|
||||
fail "Key mismatch ($test_file_folder/$bucket_file_one, $key_one)"
|
||||
fi
|
||||
if [[ "$test_file_folder/$bucket_file_two" != *"$key_two" ]]; then
|
||||
fail "Key mismatch ($test_file_folder/$bucket_file_two, $key_two)"
|
||||
fi
|
||||
|
||||
delete_bucket_or_contents "$BUCKET_ONE_NAME"
|
||||
delete_test_files "$bucket_file_one" "$bucket_file_two"
|
||||
}
|
||||
|
||||
@@ -15,7 +15,8 @@ setup() {
|
||||
fi
|
||||
else
|
||||
echo "$VERSITYGW_TEST_ENV"
|
||||
source $VERSITYGW_TEST_ENV
|
||||
# shellcheck source=./.env.default
|
||||
source "$VERSITYGW_TEST_ENV"
|
||||
fi
|
||||
|
||||
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
|
||||
@@ -42,6 +43,18 @@ setup() {
|
||||
elif [ -z "$AWS_ENDPOINT_URL" ]; then
|
||||
echo "No AWS endpoint URL set"
|
||||
return 1
|
||||
elif [ -z "$BUCKET_ONE_NAME" ]; then
|
||||
echo "No bucket one name set"
|
||||
return 1
|
||||
elif [ -z "$BUCKET_TWO_NAME" ]; then
|
||||
echo "No bucket two name set"
|
||||
return 1
|
||||
elif [ -z "$RECREATE_BUCKETS" ]; then
|
||||
echo "No recreate buckets parameter set"
|
||||
return 1
|
||||
elif [[ $RECREATE_BUCKETS != "true" ]] && [[ $RECREATE_BUCKETS != "false" ]]; then
|
||||
echo "RECREATE_BUCKETS must be 'true' or 'false'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
ROOT_ACCESS_KEY="$AWS_ACCESS_KEY_ID" ROOT_SECRET_KEY="$AWS_SECRET_ACCESS_KEY" "$VERSITY_EXE" "$BACKEND" "$LOCAL_FOLDER" &
|
||||
@@ -50,6 +63,8 @@ setup() {
|
||||
export AWS_PROFILE
|
||||
export AWS_ENDPOINT_URL
|
||||
export LOCAL_FOLDER
|
||||
export BUCKET_ONE_NAME
|
||||
export BUCKET_TWO_NAME
|
||||
|
||||
versitygw_pid=$!
|
||||
export versitygw_pid
|
||||
720
tests/util.sh
Normal file
720
tests/util.sh
Normal file
@@ -0,0 +1,720 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
# create an AWS bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
create_bucket() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "create bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 mb s3://"$1" 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error creating bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# delete an AWS bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
delete_bucket() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rb s3://"$1" 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
if [[ "$error" == *"The specified bucket does not exist"* ]]; then
|
||||
return 0
|
||||
else
|
||||
echo "error deleting bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# recursively delete an AWS bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
delete_bucket_recursive() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rb s3://"$1" --force 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
if [[ "$error" == *"The specified bucket does not exist"* ]]; then
|
||||
return 0
|
||||
else
|
||||
echo "error deleting bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# delete contents of a bucket
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
delete_bucket_contents() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete bucket missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rm s3://"$1" --recursive 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error deleting bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if bucket exists
|
||||
# param: bucket name
|
||||
# return 0 for true, 1 for false, 2 for error
|
||||
bucket_exists() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket exists check missing bucket name"
|
||||
return 2
|
||||
fi
|
||||
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
|
||||
echo "Exit code: $exit_code, error: $error"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then
|
||||
return 1
|
||||
else
|
||||
echo "error checking if bucket exists: $error"
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# delete buckets or just the contents depending on RECREATE_BUCKETS parameter
|
||||
# param: bucket name
|
||||
# return: 0 for success, 1 for failure
|
||||
delete_bucket_or_contents() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete bucket or contents function requires bucket name"
|
||||
return 1
|
||||
fi
|
||||
if [[ $RECREATE_BUCKETS != "true" ]]; then
|
||||
delete_bucket_contents "$1" || local delete_result=$?
|
||||
if [[ $delete_result -ne 0 ]]; then
|
||||
echo "error deleting bucket contents"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
delete_bucket_recursive "$1" || local delete_result=$?
|
||||
if [[ $delete_result -ne 0 ]]; then
|
||||
echo "Bucket deletion error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# if RECREATE_BUCKETS is set to true create bucket, deleting it if it exists to clear state. If not,
|
||||
# check to see if it exists and return an error if it does not.
|
||||
# param: bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
setup_bucket() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket creation function requires bucket name"
|
||||
return 1
|
||||
fi
|
||||
local exists_result
|
||||
bucket_exists "$1" || exists_result=$?
|
||||
if [[ $exists_result -eq 2 ]]; then
|
||||
echo "Bucket existence check error"
|
||||
return 1
|
||||
fi
|
||||
if [[ $exists_result -eq 0 ]]; then
|
||||
delete_bucket_or_contents "$1" || delete_result=$?
|
||||
if [[ delete_result -ne 0 ]]; then
|
||||
echo "error deleting bucket or contents"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
if [[ $RECREATE_BUCKETS != "true" ]]; then
|
||||
echo "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
|
||||
return 1
|
||||
fi
|
||||
local create_result
|
||||
create_bucket "$1" || create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
echo "Error creating bucket"
|
||||
return 1
|
||||
fi
|
||||
echo "Bucket creation success"
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if object exists on S3 via gateway
|
||||
# param: object path
|
||||
# return 0 for true, 1 for false, 2 for error
|
||||
object_exists() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "object exists check missing object name"
|
||||
return 2
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
if [[ "$error" == "" ]]; then
|
||||
return 1
|
||||
else
|
||||
echo "error checking if object exists: $error"
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# add object to versitygw
|
||||
# params: source file, destination copy location
|
||||
# return 0 for success, 1 for failure
|
||||
put_object() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "put object command requires source, destination"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 cp "$1" s3://"$2" 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error copying object to bucket: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# add object to versitygw if it doesn't exist
|
||||
# params: source file, destination copy location
|
||||
# return 0 for success or already exists, 1 for failure
|
||||
check_and_put_object() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "check and put object function requires source, destination"
|
||||
return 1
|
||||
fi
|
||||
object_exists "$2" || local exists_result=$?
|
||||
if [ "$exists_result" -eq 2 ]; then
|
||||
echo "error checking if object exists"
|
||||
return 1
|
||||
fi
|
||||
if [ "$exists_result" -eq 1 ]; then
|
||||
put_object "$1" "$2" || local put_result=$?
|
||||
if [ "$put_result" -ne 0 ]; then
|
||||
echo "error adding object"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# delete object from versitygw
|
||||
# param: object path, including bucket name
|
||||
# return 0 for success, 1 for failure
|
||||
delete_object() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "delete object command requires object parameter"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3 rm s3://"$1" 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error deleting object: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# list buckets on versitygw
|
||||
# no params
|
||||
# export bucket_array (bucket names) on success, return 1 for failure
|
||||
list_buckets() {
|
||||
local exit_code=0
|
||||
local output
|
||||
output=$(aws s3 ls 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error listing buckets: $output"
|
||||
return 1
|
||||
fi
|
||||
|
||||
bucket_array=()
|
||||
while IFS= read -r line; do
|
||||
bucket_name=$(echo "$line" | awk '{print $NF}')
|
||||
bucket_array+=("$bucket_name")
|
||||
done <<< "$output"
|
||||
|
||||
export bucket_array
|
||||
}
|
||||
|
||||
# list objects on versitygw, in bucket or folder
|
||||
# param: path of bucket or folder
|
||||
# export object_array (object names) on success, return 1 for failure
|
||||
list_objects() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "list objects command requires bucket or folder"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
local output
|
||||
output=$(aws s3 ls s3://"$1" 2>&1) || exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "error listing objects: $output"
|
||||
return 1
|
||||
fi
|
||||
|
||||
object_array=()
|
||||
while IFS= read -r line; do
|
||||
object_name=$(echo "$line" | awk '{print $NF}')
|
||||
object_array+=("$object_name")
|
||||
done <<< "$output"
|
||||
|
||||
export object_array
|
||||
}
|
||||
|
||||
# check if bucket info can be retrieved
|
||||
# param: path of bucket or folder
|
||||
# return 0 for yes, 1 for no, 2 for error
|
||||
bucket_is_accessible() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket accessibility check missing bucket name"
|
||||
return 2
|
||||
fi
|
||||
local exit_code=0
|
||||
local error
|
||||
error=$(aws s3api head-bucket --bucket "$1" 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ "$error" == *"500"* ]]; then
|
||||
return 1
|
||||
fi
|
||||
echo "Error checking bucket accessibility: $error"
|
||||
return 2
|
||||
}
|
||||
|
||||
# check if object info (etag) is accessible
|
||||
# param: path of object
|
||||
# return 0 for yes, 1 for no, 2 for error
|
||||
object_is_accessible() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "object accessibility check missing bucket and/or key"
|
||||
return 2
|
||||
fi
|
||||
local exit_code=0
|
||||
object_data=$(aws s3api head-object --bucket "$1" --key "$2" 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "Error obtaining object data: $object_data"
|
||||
return 2
|
||||
fi
|
||||
etag=$(echo "$object_data" | jq '.ETag')
|
||||
if [[ "$etag" == '""' ]]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# get bucket acl
|
||||
# param: bucket path
|
||||
# export acl for success, return 1 for error
|
||||
get_bucket_acl() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket ACL command missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
local exit_code=0
|
||||
acl=$(aws s3api get-bucket-acl --bucket "$1" 2>&1) || exit_code="$?"
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "Error getting bucket ACLs: $acl"
|
||||
return 1
|
||||
fi
|
||||
export acl
|
||||
}
|
||||
|
||||
# add tags to bucket
|
||||
# params: bucket, key, value
|
||||
# return: 0 for success, 1 for error
|
||||
put_bucket_tag() {
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "bucket tag command missing bucket name, key, value"
|
||||
return 1
|
||||
fi
|
||||
local error
|
||||
local result
|
||||
error=$(aws s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "Error adding bucket tag: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# get bucket tags
|
||||
# params: bucket
|
||||
# export 'tags' on success, return 1 for error
|
||||
get_bucket_tags() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "get bucket tag command missing bucket name"
|
||||
return 1
|
||||
fi
|
||||
local result
|
||||
tags=$(aws s3api get-bucket-tagging --bucket "$1") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error getting bucket tags: $tags"
|
||||
return 1
|
||||
fi
|
||||
export tags
|
||||
}
|
||||
|
||||
# add tags to object
|
||||
# params: object, key, value
|
||||
# return: 0 for success, 1 for error
|
||||
put_object_tag() {
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "object tag command missing object name, file, key, and/or value"
|
||||
return 1
|
||||
fi
|
||||
local error
|
||||
local result
|
||||
error=$(aws s3api put-object-tagging --bucket "$1" --key "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "Error adding object tag: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# get object tags
|
||||
# params: bucket
|
||||
# export 'tags' on success, return 1 for error
|
||||
get_object_tags() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "get object tag command missing bucket and/or key"
|
||||
return 1
|
||||
fi
|
||||
local result
|
||||
tags=$(aws s3api get-object-tagging --bucket "$1" --key "$2") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error getting object tags: $tags"
|
||||
return 1
|
||||
fi
|
||||
export tags
|
||||
}
|
||||
|
||||
# create a test file and export folder. do so in temp folder
|
||||
# params: filename
|
||||
# export test file folder on success, return 1 for error
|
||||
create_test_files() {
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "create test files command missing filename"
|
||||
return 1
|
||||
fi
|
||||
test_file_folder=.
|
||||
if [[ -z "$GITHUB_ACTIONS" ]]; then
|
||||
test_file_folder=${TMPDIR}versity-gwtest
|
||||
mkdir -p "$test_file_folder" || local mkdir_result=$?
|
||||
if [[ $mkdir_result -ne 0 ]]; then
|
||||
echo "error creating test file folder"
|
||||
fi
|
||||
fi
|
||||
for name in "$@"; do
|
||||
touch "$test_file_folder"/"$name" || local touch_result=$?
|
||||
if [[ $touch_result -ne 0 ]]; then
|
||||
echo "error creating file $name"
|
||||
fi
|
||||
done
|
||||
export test_file_folder
|
||||
}
|
||||
|
||||
# delete a test file
|
||||
# params: filename
|
||||
# return: 0 for success, 1 for error
|
||||
delete_test_files() {
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "delete test files command missing filenames"
|
||||
return 1
|
||||
fi
|
||||
if [ -z "$test_file_folder" ]; then
|
||||
echo "no test file folder defined, not deleting"
|
||||
return 1
|
||||
fi
|
||||
for name in "$@"; do
|
||||
rm "$test_file_folder"/"$name" || rm_result=$?
|
||||
if [[ $rm_result -ne 0 ]]; then
|
||||
echo "error deleting file $name"
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# list objects in bucket, v1
|
||||
# param: bucket
|
||||
# export objects on success, return 1 for failure
|
||||
list_objects_s3api_v1() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "list objects command missing bucket"
|
||||
return 1
|
||||
fi
|
||||
objects=$(aws s3api list-objects --bucket "$1") || local result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error listing objects: $objects"
|
||||
return 1
|
||||
fi
|
||||
export objects
|
||||
}
|
||||
|
||||
# list objects in bucket, v2
|
||||
# param: bucket
|
||||
# export objects on success, return 1 for failure
|
||||
list_objects_s3api_v2() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "list objects command missing bucket and/or path"
|
||||
return 1
|
||||
fi
|
||||
objects=$(aws s3api list-objects-v2 --bucket "$1") || local result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error listing objects: $objects"
|
||||
return 1
|
||||
fi
|
||||
export objects
|
||||
}
|
||||
|
||||
# initialize a multipart upload
|
||||
# params: bucket, key
|
||||
# return 0 for success, 1 for failure
|
||||
create_multipart_upload() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "create multipart upload function must have bucket, key"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local multipart_data
|
||||
multipart_data=$(aws s3api create-multipart-upload --bucket "$1" --key "$2") || local created=$?
|
||||
if [[ $created -ne 0 ]]; then
|
||||
echo "Error creating multipart upload: $upload_id"
|
||||
return 1
|
||||
fi
|
||||
|
||||
upload_id=$(echo "$multipart_data" | jq '.UploadId')
|
||||
upload_id="${upload_id//\"/}"
|
||||
export upload_id
|
||||
}
|
||||
|
||||
# upload a single part of a multipart upload
|
||||
# params: bucket, key, upload ID, original (unsplit) file name, part number
|
||||
# return: 0 for success, 1 for failure
|
||||
upload_part() {
|
||||
if [ $# -ne 5 ]; then
|
||||
echo "upload multipart part function must have bucket, key, upload ID, file name, part number"
|
||||
return 1
|
||||
fi
|
||||
local etag_json
|
||||
etag_json=$(aws s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))") || local uploaded=$?
|
||||
if [[ $uploaded -ne 0 ]]; then
|
||||
echo "Error uploading part $5: $etag_json"
|
||||
return 1
|
||||
fi
|
||||
etag=$(echo "$etag_json" | jq '.ETag')
|
||||
export etag
|
||||
}
|
||||
|
||||
# perform all parts of a multipart upload before completion command
|
||||
# params: bucket, key, file to split and upload, number of file parts to upload
|
||||
# return: 0 for success, 1 for failure
|
||||
multipart_upload_before_completion() {
|
||||
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "multipart upload pre-completion command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
file_size=$(stat -c %s "$3" 2>/dev/null || stat -f %z "$3" 2>/dev/null)
|
||||
part_size=$((file_size / $4))
|
||||
remainder=$((file_size % $4))
|
||||
if [[ remainder -ne 0 ]]; then
|
||||
part_size=$((part_size+1))
|
||||
fi
|
||||
local error
|
||||
local split_result
|
||||
error=$(split -a 1 -d -b "$part_size" "$3" "$3"-) || split_result=$?
|
||||
if [[ $split_result -ne 0 ]]; then
|
||||
echo "error splitting file: $error"
|
||||
return 1
|
||||
fi
|
||||
|
||||
create_multipart_upload "$1" "$2" || create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
echo "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
upload_part "$1" "$2" "$upload_id" "$3" "$i" || local upload_result=$?
|
||||
if [[ $upload_result -ne 0 ]]; then
|
||||
echo "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
# perform a multi-part upload
|
||||
# params: bucket, key, source file location, number of parts
|
||||
# return 0 for success, 1 for failure
|
||||
multipart_upload() {
|
||||
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "multipart upload command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
error=$(aws s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}') || local completed=$?
|
||||
if [[ $completed -ne 0 ]]; then
|
||||
echo "Error completing upload: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# run the abort multipart command
|
||||
# params: bucket, key, upload ID
|
||||
# return 0 for success, 1 for failure
|
||||
run_abort_command() {
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "command to run abort requires bucket, key, upload ID"
|
||||
return 1
|
||||
fi
|
||||
|
||||
error=$(aws s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3") || local aborted=$?
|
||||
if [[ $aborted -ne 0 ]]; then
|
||||
echo "Error aborting upload: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# run upload, then abort it
|
||||
# params: bucket, key, local file location, number of parts to split into before uploading
|
||||
# return 0 for success, 1 for failure
|
||||
abort_multipart_upload() {
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "abort multipart upload command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
run_abort_command "$1" "$2" "$upload_id"
|
||||
return $?
|
||||
}
|
||||
|
||||
# copy a file to/from S3
|
||||
# params: source, destination
|
||||
# return 0 for success, 1 for failure
|
||||
copy_file() {
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "copy file command requires src and dest"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local result
|
||||
error=$(aws s3 cp "$1" "$2") || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error copying file: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# list parts of an unfinished multipart upload
|
||||
# params: bucket, key, local file location, and parts to split into before upload
|
||||
# export parts on success, return 1 for error
|
||||
list_parts() {
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "list multipart upload parts command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
echo "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
listed_parts=$(aws s3api list-parts --bucket "$1" --key "$2" --upload-id "$upload_id") || local listed=$?
|
||||
if [[ $listed -ne 0 ]]; then
|
||||
echo "Error aborting upload: $parts"
|
||||
return 1
|
||||
fi
|
||||
export listed_parts
|
||||
}
|
||||
|
||||
# list unfinished multipart uploads
|
||||
# params: bucket, key one, key two
|
||||
# export current two uploads on success, return 1 for error
|
||||
list_multipart_uploads() {
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "list multipart uploads command requires bucket and two keys"
|
||||
return 1
|
||||
fi
|
||||
|
||||
create_multipart_upload "$1" "$2" || local create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
echo "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
create_multipart_upload "$1" "$3" || local create_result_two=$?
|
||||
if [[ $create_result_two -ne 0 ]]; then
|
||||
echo "error creating multpart upload two"
|
||||
return 1
|
||||
fi
|
||||
|
||||
uploads=$(aws s3api list-multipart-uploads --bucket "$1") || local list_result=$?
|
||||
if [[ $list_result -ne 0 ]]; then
|
||||
echo "error listing uploads: $uploads"
|
||||
return 1
|
||||
fi
|
||||
export uploads
|
||||
}
|
||||
|
||||
97
tests/util_posix.sh
Normal file
97
tests/util_posix.sh
Normal file
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
# check if object exists both on S3 and locally
|
||||
# param: object path
|
||||
# 0 for yes, 1 for no, 2 for error
|
||||
object_exists_remote_and_local() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "object existence check requires single name parameter"
|
||||
return 2
|
||||
fi
|
||||
object_exists "$1" || local exist_result=$?
|
||||
if [[ $exist_result -eq 2 ]]; then
|
||||
echo "Error checking if object exists"
|
||||
return 2
|
||||
fi
|
||||
if [[ $exist_result -eq 1 ]]; then
|
||||
echo "Error: object doesn't exist remotely"
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then
|
||||
echo "Error: object doesn't exist locally"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if object doesn't exist both on S3 and locally
|
||||
# param: object path
|
||||
# return 0 for doesn't exist, 1 for still exists, 2 for error
|
||||
object_not_exists_remote_and_local() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "object non-existence check requires single name parameter"
|
||||
return 2
|
||||
fi
|
||||
object_exists "$1" || local exist_result=$?
|
||||
if [[ $exist_result -eq 2 ]]; then
|
||||
echo "Error checking if object doesn't exist"
|
||||
return 2
|
||||
fi
|
||||
if [[ $exist_result -eq 0 ]]; then
|
||||
echo "Error: object exists remotely"
|
||||
return 1
|
||||
fi
|
||||
if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then
|
||||
echo "Error: object exists locally"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if a bucket doesn't exist both on S3 and on gateway
|
||||
# param: bucket name
|
||||
# return: 0 for doesn't exist, 1 for does, 2 for error
|
||||
bucket_not_exists_remote_and_local() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket existence check requires single name parameter"
|
||||
return 2
|
||||
fi
|
||||
bucket_exists "$1" || local exist_result=$?
|
||||
if [[ $exist_result -eq 2 ]]; then
|
||||
echo "Error checking if bucket exists"
|
||||
return 2
|
||||
fi
|
||||
if [[ $exist_result -eq 0 ]]; then
|
||||
echo "Error: bucket exists remotely"
|
||||
return 1
|
||||
fi
|
||||
if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then
|
||||
echo "Error: bucket exists locally"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# check if a bucket exists both on S3 and on gateway
|
||||
# param: bucket name
|
||||
# return: 0 for yes, 1 for no, 2 for error
|
||||
bucket_exists_remote_and_local() {
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "bucket existence check requires single name parameter"
|
||||
return 2
|
||||
fi
|
||||
bucket_exists "$1" || local exist_result=$?
|
||||
if [[ $exist_result -eq 2 ]]; then
|
||||
echo "Error checking if bucket exists"
|
||||
return 2
|
||||
fi
|
||||
if [[ $exist_result -eq 1 ]]; then
|
||||
echo "Error: bucket doesn't exist remotely"
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then
|
||||
echo "Error: bucket doesn't exist locally"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
Reference in New Issue
Block a user