feat: moved to golang s3 v2 sdk

2026-04-22 21:50:29 +00:00 · 2023-05-17 00:47:03 +05:00
parent 346a05b49a
commit bbba9413ff
3 changed files with 68 additions and 68 deletions
--- a/backend/backend.go
+++ b/backend/backend.go
@@ -2,11 +2,11 @@ package backend

 import (
 	"fmt"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"io"

-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/versity/scoutgw/s3err"
-	"github.com/versity/scoutgw/s3response"
 )

 type Backend interface {
@@ -14,30 +14,30 @@ type Backend interface {
 	GetIAMConfig() ([]byte, error)
 	Shutdown()

-	ListBuckets() (*s3response.ListAllMyBucketsList, s3err.ErrorCode)
-	HeadBucket(bucket string) (*s3response.HeadBucketResponse, s3err.ErrorCode)
-	GetBucketAcl(bucket string) (*s3response.GetBucketAclResponse, s3err.ErrorCode)
+	ListBuckets() (*s3.ListBucketsOutput, s3err.ErrorCode)
+	HeadBucket(bucket string) (*s3.HeadBucketOutput, s3err.ErrorCode)
+	GetBucketAcl(bucket string) (*s3.GetBucketAclOutput, s3err.ErrorCode)
 	PutBucket(bucket string) s3err.ErrorCode
 	DeleteBucket(bucket string) s3err.ErrorCode

-	CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3response.InitiateMultipartUploadResponse, s3err.ErrorCode)
-	CompleteMultipartUpload(bucket, object, uploadID string, parts []s3response.Part) (*s3response.CompleteMultipartUploadResponse, s3err.ErrorCode)
+	CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, s3err.ErrorCode)
+	CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, s3err.ErrorCode)
 	AbortMultipartUpload(*s3.AbortMultipartUploadInput) s3err.ErrorCode
-	ListMultipartUploads(*s3response.ListMultipartUploads) (*s3response.ListMultipartUploadsResponse, s3err.ErrorCode)
-	ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3response.ListPartsResponse, s3err.ErrorCode)
-	CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*s3response.CopyObjectPartResponse, s3err.ErrorCode)
+	ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, s3err.ErrorCode)
+	ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, s3err.ErrorCode)
+	CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, s3err.ErrorCode)
 	PutObjectPart(bucket, object, uploadID string, part int, r io.Reader) (etag string, err s3err.ErrorCode)

 	PutObject(bucket, object string, r io.Reader) (string, s3err.ErrorCode)
-	HeadObject(bucket, object string, etag string) (*s3response.HeadObjectResponse, s3err.ErrorCode)
-	GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3response.GetObjectResponse, s3err.ErrorCode)
-	GetObjectAcl(bucket, object string) (*s3response.GetObjectAccessControlPolicyResponse, s3err.ErrorCode)
-	GetObjectAttributes(bucket, object string, attributes []string) (*s3response.GetObjectAttributesResponse, s3err.ErrorCode)
-	CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3response.CopyObjectResponse, s3err.ErrorCode)
-	ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3response.ListBucketResult, s3err.ErrorCode)
-	ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3response.ListBucketResultV2, s3err.ErrorCode)
+	HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, s3err.ErrorCode)
+	GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3.GetObjectOutput, s3err.ErrorCode)
+	GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, s3err.ErrorCode)
+	GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, s3err.ErrorCode)
+	CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, s3err.ErrorCode)
+	ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, s3err.ErrorCode)
+	ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, s3err.ErrorCode)
 	DeleteObject(bucket, object string) s3err.ErrorCode
-	DeleteObjects(bucket string, objects *s3response.DeleteObjectsInput) s3err.ErrorCode
+	DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) s3err.ErrorCode

 	IsTaggingSupported() bool
 	GetTags(bucket, object string) (map[string]string, error)
@@ -59,14 +59,13 @@ func (BackendUnsupported) Shutdown()           {}
 func (BackendUnsupported) String() string {
 	return "Unsupported"
 }
-
-func (BackendUnsupported) ListBuckets() (*s3response.ListAllMyBucketsList, s3err.ErrorCode) {
+func (BackendUnsupported) ListBuckets() (*s3.ListBucketsOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) GetBucketAcl(bucket string) (*s3response.GetBucketAclResponse, s3err.ErrorCode) {
+func (BackendUnsupported) GetBucketAcl(bucket string) (*s3.GetBucketAclOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) HeadBucket(bucket string) (*s3response.HeadBucketResponse, s3err.ErrorCode) {
+func (BackendUnsupported) HeadBucket(bucket string) (*s3.HeadBucketOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
 func (BackendUnsupported) PutBucket(bucket string) s3err.ErrorCode {
@@ -76,56 +75,56 @@ func (BackendUnsupported) DeleteBucket(bucket string) s3err.ErrorCode {
 	return s3err.ErrNotImplemented
 }

-func (BackendUnsupported) CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3response.InitiateMultipartUploadResponse, s3err.ErrorCode) {
+func (BackendUnsupported) CreateMultipartUpload(input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID string, parts []s3response.Part) (*s3response.CompleteMultipartUploadResponse, s3err.ErrorCode) {
+func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) AbortMultipartUpload(*s3.AbortMultipartUploadInput) s3err.ErrorCode {
+func (BackendUnsupported) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) s3err.ErrorCode {
 	return s3err.ErrNotImplemented
 }
-func (BackendUnsupported) ListMultipartUploads(*s3response.ListMultipartUploads) (*s3response.ListMultipartUploadsResponse, s3err.ErrorCode) {
+func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3response.ListPartsResponse, s3err.ErrorCode) {
+func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*s3response.CopyObjectPartResponse, s3err.ErrorCode) {
+func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) PutObjectPart(buket, object, uploadID string, part int, r io.Reader) (string, s3err.ErrorCode) {
+func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, r io.Reader) (etag string, err s3err.ErrorCode) {
 	return "", s3err.ErrNotImplemented
 }

-func (BackendUnsupported) PutObject(buket, object string, r io.Reader) (string, s3err.ErrorCode) {
+func (BackendUnsupported) PutObject(bucket, object string, r io.Reader) (string, s3err.ErrorCode) {
 	return "", s3err.ErrNotImplemented
 }
 func (BackendUnsupported) DeleteObject(bucket, object string) s3err.ErrorCode {
 	return s3err.ErrNotImplemented
 }
-func (BackendUnsupported) DeleteObjects(bucket string, objects *s3response.DeleteObjectsInput) s3err.ErrorCode {
+func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) s3err.ErrorCode {
 	return s3err.ErrNotImplemented
 }
-func (BackendUnsupported) GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3response.GetObjectResponse, s3err.ErrorCode) {
+func (BackendUnsupported) GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3.GetObjectOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) HeadObject(bucket, object string, etag string) (*s3response.HeadObjectResponse, s3err.ErrorCode) {
+func (BackendUnsupported) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3response.GetObjectAccessControlPolicyResponse, s3err.ErrorCode) {
+func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes []string) (*s3response.GetObjectAttributesResponse, s3err.ErrorCode) {
+func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3response.CopyObjectResponse, s3err.ErrorCode) {
+func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3response.ListBucketResult, s3err.ErrorCode) {
+func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }
-func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3response.ListBucketResultV2, s3err.ErrorCode) {
+func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, s3err.ErrorCode) {
 	return nil, s3err.ErrNotImplemented
 }

--- a/s3api/router.go
+++ b/s3api/router.go
@@ -3,15 +3,16 @@ package s3api
 import (
 	"encoding/xml"
 	"errors"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"strconv"
 	"strings"

-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/scoutgw/backend"
 	"github.com/versity/scoutgw/internal"
 	"github.com/versity/scoutgw/s3err"
-	"github.com/versity/scoutgw/s3response"
 )

 type S3ApiRouter struct {
@@ -23,7 +24,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 	// ListBuckets action
 	app.Get("/", func(ctx *fiber.Ctx) error {
 		res, code := be.ListBuckets()
-		return responce[*s3response.ListAllMyBucketsList](ctx, res, code)
+		return responce[*s3.ListBucketsOutput](ctx, res, code)
 	})

 	// PutBucket action
@@ -40,7 +41,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 	// HeadBucket
 	app.Head("/:bucket", func(ctx *fiber.Ctx) error {
 		res, code := be.HeadBucket(ctx.Params("bucket"))
-		return responce[*s3response.HeadBucketResponse](ctx, res, code)
+		return responce[*s3.HeadBucketOutput](ctx, res, code)
 	})
 	// GetBucketAcl action
 	// ListMultipartUploads action
@@ -49,21 +50,21 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 	app.Get("/:bucket", func(ctx *fiber.Ctx) error {
 		if ctx.Request().URI().QueryArgs().Has("acl") {
 			res, code := be.GetBucketAcl(ctx.Params("bucket"))
-			return responce[*s3response.GetBucketAclResponse](ctx, res, code)
+			return responce[*s3.GetBucketAclOutput](ctx, res, code)
 		}

 		if ctx.Request().URI().QueryArgs().Has("uploads") {
-			res, code := be.ListMultipartUploads(&s3response.ListMultipartUploads{Bucket: ctx.Params("bucket")})
-			return responce[*s3response.ListMultipartUploadsResponse](ctx, res, code)
+			res, code := be.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))})
+			return responce[*s3.ListMultipartUploadsOutput](ctx, res, code)
 		}

 		if ctx.QueryInt("list-type") == 2 {
 			res, code := be.ListObjectsV2(ctx.Params("bucket"), "", "", "", 1)
-			return responce[*s3response.ListBucketResultV2](ctx, res, code)
+			return responce[*s3.ListBucketsOutput](ctx, res, code)
 		}

 		res, code := be.ListObjects(ctx.Params("bucket"), "", "", "", 1)
-		return responce[*s3response.ListBucketResult](ctx, res, code)
+		return responce[*s3.ListBucketsOutput](ctx, res, code)
 	})

 	// HeadObject action
@@ -74,7 +75,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 		}

 		res, code := be.HeadObject(bucket, key, "")
-		return responce[*s3response.HeadObjectResponse](ctx, res, code)
+		return responce[*s3.HeadObjectOutput](ctx, res, code)
 	})
 	// GetObjectAcl action
 	// GetObject action
@@ -97,17 +98,17 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 			}

 			res, code := be.ListObjectParts(bucket, "", uploadId, partNumberMarker, maxParts)
-			return responce[*s3response.ListPartsResponse](ctx, res, code)
+			return responce[*s3.ListPartsOutput](ctx, res, code)
 		}

 		if ctx.Request().URI().QueryArgs().Has("acl") {
 			res, code := be.GetObjectAcl(bucket, key)
-			return responce[*s3response.GetObjectAccessControlPolicyResponse](ctx, res, code)
+			return responce[*s3.GetObjectAclOutput](ctx, res, code)
 		}

 		if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" {
 			res, code := be.GetObjectAttributes(bucket, key, strings.Split(attrs, ","))
-			return responce[*s3response.GetObjectAttributesResponse](ctx, res, code)
+			return responce[*s3.GetObjectAttributesOutput](ctx, res, code)
 		}

 		bRangeSl := strings.Split(ctx.Get("Range"), "=")
@@ -131,7 +132,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 		}

 		res, code := be.GetObject(bucket, key, int64(startOffset), int64(length), ctx.Response().BodyWriter(), "")
-		return responce[*s3response.GetObjectResponse](ctx, res, code)
+		return responce[*s3.GetObjectOutput](ctx, res, code)
 	})
 	// DeleteObject action
 	// AbortMultipartUpload action
@@ -146,11 +147,11 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 			expectedBucketOwner, requestPayer := ctx.Get("X-Amz-Expected-Bucket-Owner"), ctx.Get("X-Amz-Request-Payer")

 			code := be.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
-				UploadId: &uploadId, 
-				Bucket: &bucket, 
-				Key: &key, 
-				ExpectedBucketOwner: &expectedBucketOwner, 
-				RequestPayer: &requestPayer,
+				UploadId:            &uploadId,
+				Bucket:              &bucket,
+				Key:                 &key,
+				ExpectedBucketOwner: &expectedBucketOwner,
+				RequestPayer:        types.RequestPayer(requestPayer),
 			})
 			return responce[internal.Any](ctx, nil, code)
 		}
@@ -160,35 +161,35 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 	})
 	// DeleteObjects action
 	app.Post("/:bucket", func(ctx *fiber.Ctx) error {
-		var dObj s3response.DeleteObjectEntry
+		var dObj types.Delete
 		if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
 			return errors.New("wrong api call")
 		}

-		code := be.DeleteObjects(ctx.Params("bucket"), &s3response.DeleteObjectsInput{Delete: dObj})
+		code := be.DeleteObjects(ctx.Params("bucket"), &s3.DeleteObjectsInput{Delete: &dObj})
 		return responce[internal.Any](ctx, nil, code)
 	})
 	// CompleteMultipartUpload action
 	// CreateMultipartUpload
 	app.Post("/:bucket/:key/*", func(ctx *fiber.Ctx) error {
 		bucket, key, keyEnd, uploadId := ctx.Params("bucket"), ctx.Params("key"), ctx.Params("*1"), ctx.Query("uploadId")
-		
+
 		if keyEnd != "" {
 			key = strings.Join([]string{key, keyEnd}, "/")
 		}

 		if uploadId != "" {
-			var parts []s3response.Part
+			var parts []types.Part

 			if err := xml.Unmarshal(ctx.Body(), &parts); err != nil {
 				return errors.New("wrong api call")
 			}
-			
-			res, code := be.CompleteMultipartUpload(bucket, "", uploadId,  parts)
-			return responce[*s3response.CompleteMultipartUploadResponse](ctx, res, code)
+
+			res, code := be.CompleteMultipartUpload(bucket, "", uploadId, parts)
+			return responce[*s3.CompleteMultipartUploadOutput](ctx, res, code)
 		}
 		res, code := be.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: &bucket, Key: &key})
-		return responce[*s3response.InitiateMultipartUploadResponse](ctx, res, code)
+		return responce[*s3.CreateMultipartUploadOutput](ctx, res, code)
 	})
 	// CopyObject action
 	app.Put("/:bucket/:key/*", func(ctx *fiber.Ctx) error {
@@ -204,7 +205,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
 		}

 		res, code := be.CopyObject(srcBucket, strings.Join(srcObject, "/"), dstBucket, dstKeyStart)
-		return responce[*s3response.CopyObjectResponse](ctx, res, code)
+		return responce[*s3.CopyObjectOutput](ctx, res, code)
 	})
 }

--- a/s3api/server.go
+++ b/s3api/server.go
@@ -3,10 +3,10 @@ package s3api
 import (
 	"encoding/xml"
 	"fmt"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/logger"
 	"github.com/versity/scoutgw/backend"
-	"github.com/versity/scoutgw/s3response"
 )

 type S3ApiServer struct {
@@ -25,7 +25,7 @@ func New(app *fiber.App, be backend.Backend, port string) (s3ApiServer *S3ApiSer
 	app.All("/*", func(ctx *fiber.Ctx) error {

 		fmt.Println(ctx.Method())
-		listBucket := new(s3response.ListBucket)
+		listBucket := new(s3.ListBucketsInput)
 		if b, err := xml.Marshal(listBucket); err != nil {
 			return err
 		} else {