fix: Up to date with main

backend/backend.go

@@ -51,7 +51,7 @@ type Backend interface {
 	GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error)
 	GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
 	GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error)
-	CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error)
+	CopyObject(srcBucket, srcObject, dstBucket, dstObject string) (*s3.CopyObjectOutput, error)
 	ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error)
 	ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error)
 	DeleteObject(bucket, object string) error

backend/common.go

@@ -17,7 +17,6 @@ package backend
 import (
 	"crypto/md5"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"io/fs"
 	"strconv"
@@ -25,6 +24,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
 
@@ -55,6 +55,12 @@ func GetTimePtr(t time.Time) *time.Time {
 	return &t
 }
 
+var (
+	errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRequest)
+)
+
+// ParseRange parses input range header and returns startoffset, length, and
+// error. If no endoffset specified, then length is set to -1.
 func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
 	if acceptRange == "" {
 		return 0, file.Size(), nil
@@ -63,29 +69,34 @@ func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
 	rangeKv := strings.Split(acceptRange, "=")
 
 	if len(rangeKv) < 2 {
-		return 0, 0, errors.New("invalid range parameter")
+		return 0, 0, errInvalidRange
 	}
 
 	bRange := strings.Split(rangeKv[1], "-")
-	if len(bRange) < 2 {
-		return 0, 0, errors.New("invalid range parameter")
+	if len(bRange) < 1 || len(bRange) > 2 {
+		return 0, 0, errInvalidRange
 	}
 
 	startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
 	if err != nil {
-		return 0, 0, errors.New("invalid range parameter")
+		return 0, 0, errInvalidRange
 	}
 
-	endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
+	endOffset := int64(-1)
+	if len(bRange) == 1 || bRange[1] == "" {
+		return startOffset, endOffset, nil
+	}
+
+	endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
 	if err != nil {
-		return 0, 0, errors.New("invalid range parameter")
+		return 0, 0, errInvalidRange
 	}
 
 	if endOffset < startOffset {
-		return 0, 0, errors.New("invalid range parameter")
+		return 0, 0, errInvalidRange
 	}
 
-	return int64(startOffset), int64(endOffset - startOffset + 1), nil
+	return startOffset, endOffset - startOffset + 1, nil
 }
 
 func GetMultipartMD5(parts []types.Part) string {

backend/posix/posix.go

@@ -891,8 +891,11 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, writer io.Writer)
 		return nil, err
 	}
 
-	if startOffset+length > fi.Size() {
-		// TODO: is ErrInvalidRequest correct here?
+	if length == -1 {
+		length = fi.Size() - startOffset + 1
+	}
+
+	if startOffset+length > fi.Size()+1 {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 
@@ -975,7 +978,7 @@ func (p *Posix) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error)
 	}, nil
 }
 
-func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
+func (p *Posix) CopyObject(srcBucket, srcObject, dstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
 	_, err := os.Stat(srcBucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -984,7 +987,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
 		return nil, fmt.Errorf("stat bucket: %w", err)
 	}
 
-	_, err = os.Stat(DstBucket)
+	_, err = os.Stat(dstBucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
 	}
@@ -1002,12 +1005,17 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
 	}
 	defer f.Close()
 
-	etag, err := p.PutObject(&s3.PutObjectInput{Bucket: &DstBucket, Key: &dstObject, Body: f})
+	fInfo, err := f.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("stat object: %w", err)
+	}
+
+	etag, err := p.PutObject(&s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size()})
 	if err != nil {
 		return nil, err
 	}
 
-	fi, err := os.Stat(filepath.Join(DstBucket, dstObject))
+	fi, err := os.Stat(filepath.Join(dstBucket, dstObject))
 	if err != nil {
 		return nil, fmt.Errorf("stat dst object: %w", err)
 	}

backend/scoutfs/scoutfs.go

@@ -440,8 +440,11 @@ func (s *ScoutFS) GetObject(bucket, object, acceptRange string, writer io.Writer
 		return nil, err
 	}
 
+	if length == -1 {
+		length = fi.Size() - startOffset + 1
+	}
+
 	if startOffset+length > fi.Size() {
-		// TODO: is ErrInvalidRequest correct here?
 		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
 	}
 

cmd/versitygw/test.go

@@ -68,17 +68,20 @@ func initTestFlags() []cli.Flag {
 func initTestCommands() []*cli.Command {
 	return []*cli.Command{
 		{
-			Name:  "make-bucket",
-			Usage: "Test bucket creation.",
+			Name:  "bucket-actions",
+			Usage: "Test bucket creation, checking the existence, deletes it.",
 			Description: `Calls s3 gateway create-bucket action to create a new bucket,
-		then calls delete-bucket action to delete the bucket.`,
+		calls head-bucket action to check the existence, then calls delete-bucket action to delete the bucket.`,
 			Action: getAction(integration.TestMakeBucket),
 		},
 		{
-			Name:  "put-get-object",
-			Usage: "Test put & get object.",
+			Name:  "object-actions",
+			Usage: "Test put/get/delete/copy objects.",
 			Description: `Creates a bucket with s3 gateway action, puts an object in it,
-			gets the object from the bucket, deletes both the object and bucket.`,
+			tries to copy into another bucket, that doesn't exist, creates the destination bucket for copying,
+			copies the object, get's the object to check the length and content, 
+			get's the copied object to check the length and content, deletes all the objects inside the source bucket, 
+			deletes both the objects and buckets.`,
 			Action: getAction(integration.TestPutGetObject),
 		},
 		{

integration/tests.go

@@ -22,20 +22,34 @@ var (
 )
 
 func TestMakeBucket(s *S3Conf) {
-	testname := "test make bucket"
+	testname := "test make/head/delete bucket"
 	runF(testname)
 
+	s3client := s3.NewFromConfig(s.Config())
+
 	bucket := "testbucket"
 
-	err := setup(s, bucket)
+	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+	_, err := s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket})
+	cancel()
+	if err == nil {
+		failF("%v: expected error, instead got success response", testname)
+		return
+	}
+
+	err = setup(s, bucket)
 	if err != nil {
 		failF("%v: %v", testname, err)
 		return
 	}
-	passF(testname)
 
-	testname = "test delete empty bucket"
-	runF(testname)
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	_, err = s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket})
+	cancel()
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
 
 	err = teardown(s, bucket)
 	if err != nil {
@@ -46,10 +60,16 @@ func TestMakeBucket(s *S3Conf) {
 }
 
 func TestPutGetObject(s *S3Conf) {
-	testname := "test put/get object"
+	testname := "test put/get/delete/copy objects"
 	runF(testname)
 
 	bucket := "testbucket1"
+	dstBucket := "testdstbucket"
+	obj := "myobject"
+	obj2 := "myobject2"
+	copySource := bucket + "/" + obj
+
+	s3client := s3.NewFromConfig(s.Config())
 
 	err := setup(s, bucket)
 	if err != nil {
@@ -64,13 +84,22 @@ func TestPutGetObject(s *S3Conf) {
 	csum := sha256.Sum256(data)
 	r := bytes.NewReader(data)
 
-	name := "myobject"
-	s3client := s3.NewFromConfig(s.Config())
-
 	ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
 	_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
 		Bucket: &bucket,
-		Key:    &name,
+		Key:    &obj,
+		Body:   r,
+	})
+	cancel()
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
+
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: &bucket,
+		Key:    &obj2,
 		Body:   r,
 	})
 	cancel()
@@ -82,7 +111,7 @@ func TestPutGetObject(s *S3Conf) {
 	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
 	out, err := s3client.GetObject(ctx, &s3.GetObjectInput{
 		Bucket: &bucket,
-		Key:    &name,
+		Key:    &obj,
 	})
 	defer cancel()
 	if err != nil {
@@ -108,11 +137,101 @@ func TestPutGetObject(s *S3Conf) {
 		return
 	}
 
+	// Expected error: destination bucket doesn't exist
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: &dstBucket, Key: &obj, CopySource: &copySource})
+	cancel()
+	if err == nil {
+		failF("%v: expect bucket not found error instead got success response", testname)
+		return
+	}
+
+	err = setup(s, dstBucket)
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
+
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: &dstBucket, Key: &obj, CopySource: &copySource})
+	cancel()
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
+
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	copyObjOut, err := s3client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: &dstBucket,
+		Key:    &obj,
+	})
+	defer cancel()
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
+	defer copyObjOut.Body.Close()
+
+	if copyObjOut.ContentLength != int64(datalen) {
+		failF("%v: content length got %v expected %v", testname, copyObjOut.ContentLength, datalen)
+		return
+	}
+
+	b, err = io.ReadAll(copyObjOut.Body)
+	if err != nil {
+		failF("%v: read body %v", testname, err)
+		return
+	}
+
+	copysum := sha256.Sum256(b)
+	if csum != copysum {
+		failF("%v: copied object checksum got %x expected %x", testname, copysum, csum)
+		return
+	}
+
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	_, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: &bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: &obj}, {Key: &obj2}}}})
+	cancel()
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
+
+	objCount := 0
+
+	in := &s3.ListObjectsV2Input{Bucket: &bucket}
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.ListObjectsV2(ctx, in)
+		cancel()
+		if err != nil {
+			failF("%v: %v", testname, err)
+			return
+		}
+		objCount += len(out.Contents)
+		if out.IsTruncated {
+			in.ContinuationToken = out.ContinuationToken
+		} else {
+			break
+		}
+	}
+
+	if objCount != 2 {
+		failF("%v: expected object count %v instead got %v", testname, 2, objCount)
+	}
+
 	err = teardown(s, bucket)
 	if err != nil {
 		failF("%v: %v", testname, err)
 		return
 	}
+
+	err = teardown(s, dstBucket)
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
+
 	passF(testname)
 }
 
@@ -923,7 +1042,34 @@ func TestRangeGet(s *S3Conf) {
 	}
 
 	// bytes range is inclusive, go range for second value is not
-	if !isSame(b, data[100:201]) {
+	if !isEqual(b, data[100:201]) {
+		failF("%v: data mismatch of range", testname)
+		return
+	}
+
+	rangeString = "bytes=100-"
+
+	ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+	out, err = s3client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &name,
+		Range:  &rangeString,
+	})
+	defer cancel()
+	if err != nil {
+		failF("%v: %v", testname, err)
+		return
+	}
+	defer out.Body.Close()
+
+	b, err = io.ReadAll(out.Body)
+	if err != nil {
+		failF("%v: read body %v", testname, err)
+		return
+	}
+
+	// bytes range is inclusive, go range for second value is not
+	if !isEqual(b, data[100:]) {
 		failF("%v: data mismatch of range", testname)
 		return
 	}

integration/utils.go

@@ -111,31 +111,26 @@ func containsPart(part int32, list []types.Part) bool {
 	return false
 }
 
-func isSame(a, b []byte) bool {
-	if len(a) != len(b) {
+func areTagsSame(tags1, tags2 []types.Tag) bool {
+	if len(tags1) != len(tags2) {
 		return false
 	}
-	for i, x := range a {
-		if x != b[i] {
+
+	for _, tag := range tags1 {
+		if !containsTag(tag, tags2) {
 			return false
 		}
 	}
 	return true
 }
 
-// Checks if the slices contain the same objects, if the objects doesn't
-// contain map, slice, channel.
-func areTagsSame(tags1, tags2 []types.Tag) bool {
-	if len(tags1) != len(tags2) {
-		return false
-	}
-
-	for i, tag := range tags1 {
-		if *tag.Key != *tags2[i].Key || *tag.Value != *tags2[i].Value {
-			return false
+func containsTag(tag types.Tag, list []types.Tag) bool {
+	for _, item := range list {
+		if *item.Key == *tag.Key && *item.Value == *tag.Value {
+			return true
 		}
 	}
-	return true
+	return false
 }
 
 func checkGrants(grts1, grts2 []types.Grant) bool {