fix: set content type/encoding on put/mutlipart object

This fixes put object with setting a content type. If no content
type is set, then we will return a default content type for
following requests. Mutli-part upload appears to be ok.

Also fixed content eincoding and multipart uploads.

Fixes #783

backend/posix/posix.go

@@ -86,6 +86,9 @@ const (
 	objectRetentionKey  = "object-retention"
 	objectLegalHoldKey  = "object-legal-hold"
 
+	// this is the media type for directories in AWS and Nextcloud
+	dirContentType = "application/x-directory"
+
 	doFalloc   = true
 	skipFalloc = false
 )
@@ -474,7 +477,8 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa
 	}
 
 	// set content-type
-	if *mpu.ContentType != "" {
+	ctype := getString(mpu.ContentType)
+	if ctype != "" {
 		err := p.meta.StoreAttribute(bucket, filepath.Join(objdir, uploadID),
 			contentTypeHdr, []byte(*mpu.ContentType))
 		if err != nil {
@@ -485,6 +489,19 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa
 		}
 	}
 
+	// set content-encoding
+	cenc := getString(mpu.ContentEncoding)
+	if cenc != "" {
+		err := p.meta.StoreAttribute(bucket, filepath.Join(objdir, uploadID), contentEncHdr,
+			[]byte(*mpu.ContentEncoding))
+		if err != nil {
+			// cleanup object if returning error
+			os.RemoveAll(filepath.Join(tmppath, uploadID))
+			os.Remove(tmppath)
+			return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("set content-encoding: %w", err)
+		}
+	}
+
 	// set object legal hold
 	if mpu.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
 		if err := p.PutObjectLegalHold(ctx, bucket, filepath.Join(objdir, uploadID), "", true); err != nil {
@@ -649,7 +666,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 
 	userMetaData := make(map[string]string)
 	upiddir := filepath.Join(objdir, uploadID)
-	cType, _ := p.loadUserMetaData(bucket, upiddir, userMetaData)
+	cType, cEnc := p.loadUserMetaData(bucket, upiddir, userMetaData)
 
 	objname := filepath.Join(bucket, object)
 	dir := filepath.Dir(objname)
@@ -696,6 +713,15 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 		}
 	}
 
+	// set content-encoding
+	if cEnc != "" {
+		if err := p.meta.StoreAttribute(bucket, object, contentEncHdr, []byte(cEnc)); err != nil {
+			// cleanup object
+			os.Remove(objname)
+			return nil, fmt.Errorf("set object content encoding: %w", err)
+		}
+	}
+
 	// load and set legal hold
 	lHold, err := p.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
 	if err == nil {
@@ -796,15 +822,9 @@ func (p *Posix) loadUserMetaData(bucket, object string, m map[string]string) (st
 	var contentType, contentEncoding string
 	b, _ := p.meta.RetrieveAttribute(bucket, object, contentTypeHdr)
 	contentType = string(b)
-	if contentType != "" {
-		m[contentTypeHdr] = contentType
-	}
 
 	b, _ = p.meta.RetrieveAttribute(bucket, object, contentEncHdr)
 	contentEncoding = string(b)
-	if contentEncoding != "" {
-		m[contentEncHdr] = contentEncoding
-	}
 
 	return contentType, contentEncoding
 }
@@ -1408,7 +1428,8 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 		}
 
 		// set etag attribute to signify this dir was specifically put
-		err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey, []byte(emptyMD5))
+		err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey,
+			[]byte(emptyMD5))
 		if err != nil {
 			return "", fmt.Errorf("set etag attr: %w", err)
 		}
@@ -1478,7 +1499,8 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 
 	// Set object legal hold
 	if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
-		if err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true); err != nil {
+		err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true)
+		if err != nil {
 			return "", err
 		}
 	}
@@ -1493,7 +1515,8 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 		if err != nil {
 			return "", fmt.Errorf("parse object lock retention: %w", err)
 		}
-		if err := p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed); err != nil {
+		err = p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed)
+		if err != nil {
 			return "", err
 		}
 	}
@@ -1505,6 +1528,24 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 		return "", fmt.Errorf("set etag attr: %w", err)
 	}
 
+	ctype := getString(po.ContentType)
+	if ctype != "" {
+		err := p.meta.StoreAttribute(*po.Bucket, *po.Key, contentTypeHdr,
+			[]byte(*po.ContentType))
+		if err != nil {
+			return "", fmt.Errorf("set content-type attr: %w", err)
+		}
+	}
+
+	cenc := getString(po.ContentEncoding)
+	if cenc != "" {
+		err := p.meta.StoreAttribute(*po.Bucket, *po.Key, contentEncHdr,
+			[]byte(*po.ContentEncoding))
+		if err != nil {
+			return "", fmt.Errorf("set content-encoding attr: %w", err)
+		}
+	}
+
 	return etag, nil
 }
 
@@ -1697,7 +1738,8 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO
 	if fi.IsDir() {
 		userMetaData := make(map[string]string)
 
-		contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
+		_, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
+		contentType := dirContentType
 
 		b, err := p.meta.RetrieveAttribute(bucket, object, etagkey)
 		etag := string(b)
@@ -1856,8 +1898,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
 	contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData)
 
 	if fi.IsDir() {
-		// this is the media type for directories in AWS and Nextcloud
-		contentType = "application/x-directory"
+		contentType = dirContentType
 	}
 
 	b, err := p.meta.RetrieveAttribute(bucket, object, etagkey)

s3api/controllers/base.go

@@ -1530,6 +1530,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	versionId := ctx.Query("versionId")
 	acct := ctx.Locals("account").(auth.Account)
 	isRoot := ctx.Locals("isRoot").(bool)
+	contentType := ctx.Get("Content-Type")
+	contentEncoding := ctx.Get("Content-Encoding")
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	tagging := ctx.Get("x-amz-tagging")
 
@@ -2235,6 +2237,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			Bucket:                    &bucket,
 			Key:                       &keyStart,
 			ContentLength:             &contentLength,
+			ContentType:               &contentType,
+			ContentEncoding:           &contentEncoding,
 			Metadata:                  metadata,
 			Body:                      body,
 			Tagging:                   &tagging,
@@ -2842,6 +2846,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 	isRoot := ctx.Locals("isRoot").(bool)
 	parsedAcl := ctx.Locals("parsedAcl").(auth.ACL)
 	contentType := ctx.Get("Content-Type")
+	contentEncoding := ctx.Get("Content-Encoding")
 	tagging := ctx.Get("X-Amz-Tagging")
 
 	if keyEnd != "" {
@@ -3071,6 +3076,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			Key:                       &key,
 			Tagging:                   &tagging,
 			ContentType:               &contentType,
+			ContentEncoding:           &contentEncoding,
 			ObjectLockRetainUntilDate: &objLockState.RetainUntilDate,
 			ObjectLockMode:            objLockState.ObjectLockMode,
 			ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus,

tests/integration/group-tests.go

@@ -145,6 +145,7 @@ func TestHeadObject(s *S3Conf) {
 	HeadObject_mp_success(s)
 	HeadObject_non_existing_dir_object(s)
 	HeadObject_name_too_long(s)
+	HeadObject_with_contenttype(s)
 	HeadObject_success(s)
 }
 
@@ -161,6 +162,7 @@ func TestGetObject(s *S3Conf) {
 	GetObject_invalid_ranges(s)
 	GetObject_with_meta(s)
 	GetObject_success(s)
+	GetObject_directory_success(s)
 	GetObject_by_range_success(s)
 	GetObject_by_range_resp_status(s)
 	GetObject_non_existing_dir_object(s)
@@ -596,6 +598,7 @@ func GetIntTests() IntTests {
 		"HeadObject_mp_success":                                               HeadObject_mp_success,
 		"HeadObject_non_existing_dir_object":                                  HeadObject_non_existing_dir_object,
 		"HeadObject_name_too_long":                                            HeadObject_name_too_long,
+		"HeadObject_with_contenttype":                                         HeadObject_with_contenttype,
 		"HeadObject_success":                                                  HeadObject_success,
 		"GetObjectAttributes_non_existing_bucket":                             GetObjectAttributes_non_existing_bucket,
 		"GetObjectAttributes_non_existing_object":                             GetObjectAttributes_non_existing_object,
@@ -606,6 +609,7 @@ func GetIntTests() IntTests {
 		"GetObject_invalid_ranges":                                            GetObject_invalid_ranges,
 		"GetObject_with_meta":                                                 GetObject_with_meta,
 		"GetObject_success":                                                   GetObject_success,
+		"GetObject_directory_success":                                         GetObject_directory_success,
 		"GetObject_by_range_success":                                          GetObject_by_range_success,
 		"GetObject_by_range_resp_status":                                      GetObject_by_range_resp_status,
 		"GetObject_non_existing_dir_object":                                   GetObject_non_existing_dir_object,

tests/integration/tests.go

@@ -3028,6 +3028,60 @@ func HeadObject_non_existing_dir_object(s *S3Conf) error {
 
 const defaultContentType = "binary/octet-stream"
 
+func HeadObject_with_contenttype(s *S3Conf) error {
+	testName := "HeadObject_with_contenttype"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		obj, dataLen := "my-obj", int64(1234567)
+		contentType := "text/plain"
+		contentEncoding := "gzip"
+
+		_, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{
+			Bucket:          &bucket,
+			Key:             &obj,
+			ContentType:     &contentType,
+			ContentEncoding: &contentEncoding,
+		}, s3client)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
+			Bucket: &bucket,
+			Key:    &obj,
+		})
+		defer cancel()
+		if err != nil {
+			return err
+		}
+
+		contentLength := int64(0)
+		if out.ContentLength != nil {
+			contentLength = *out.ContentLength
+		}
+		if contentLength != dataLen {
+			return fmt.Errorf("expected data length %v, instead got %v", dataLen, contentLength)
+		}
+		if out.ContentType == nil {
+			return fmt.Errorf("expected content type %v, instead got nil", contentType)
+		}
+		if *out.ContentType != contentType {
+			return fmt.Errorf("expected content type %v, instead got %v", contentType, *out.ContentType)
+		}
+		if out.ContentEncoding == nil {
+			return fmt.Errorf("expected content encoding %v, instead got nil", contentEncoding)
+		}
+		if *out.ContentEncoding != contentEncoding {
+			return fmt.Errorf("expected content encoding %v, instead got %v", contentEncoding, *out.ContentEncoding)
+		}
+		if out.StorageClass != types.StorageClassStandard {
+			return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass)
+		}
+
+		return nil
+	})
+}
+
 func HeadObject_success(s *S3Conf) error {
 	testName := "HeadObject_success"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -3036,11 +3090,13 @@ func HeadObject_success(s *S3Conf) error {
 			"key1": "val1",
 			"key2": "val2",
 		}
+		ctype := defaultContentType
 
 		_, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{
-			Bucket:   &bucket,
-			Key:      &obj,
-			Metadata: meta,
+			Bucket:      &bucket,
+			Key:         &obj,
+			Metadata:    meta,
+			ContentType: &ctype,
 		}, s3client)
 		if err != nil {
 			return err
@@ -3443,10 +3499,12 @@ func GetObject_success(s *S3Conf) error {
 	testName := "GetObject_success"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
 		dataLength, obj := int64(1234567), "my-obj"
+		ctype := defaultContentType
 
 		csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{
-			Bucket: &bucket,
-			Key:    &obj,
+			Bucket:      &bucket,
+			Key:         &obj,
+			ContentType: &ctype,
 		}, s3client)
 		if err != nil {
 			return err
@@ -3484,6 +3542,45 @@ func GetObject_success(s *S3Conf) error {
 	})
 }
 
+const directoryContentType = "application/x-directory"
+
+func GetObject_directory_success(s *S3Conf) error {
+	testName := "GetObject_directory_success"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		dataLength, obj := int64(0), "my-dir/"
+
+		_, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{
+			Bucket: &bucket,
+			Key:    &obj,
+		}, s3client)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: &bucket,
+			Key:    &obj,
+		})
+		defer cancel()
+		if err != nil {
+			return err
+		}
+		if *out.ContentLength != dataLength {
+			return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength)
+		}
+		if *out.ContentType != directoryContentType {
+			return fmt.Errorf("expected content type %v, instead got %v", directoryContentType, *out.ContentType)
+		}
+		if out.StorageClass != types.StorageClassStandard {
+			return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass)
+		}
+
+		out.Body.Close()
+		return nil
+	})
+}
+
 func GetObject_by_range_success(s *S3Conf) error {
 	testName := "GetObject_by_range_success"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -5087,11 +5184,16 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error {
 			"prop1": "val1",
 			"prop2": "val2",
 		}
+		contentType := "application/text"
+		contentEncoding := "testenc"
+
 		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
 		out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
-			Bucket:   &bucket,
-			Key:      &obj,
-			Metadata: meta,
+			Bucket:          &bucket,
+			Key:             &obj,
+			Metadata:        meta,
+			ContentType:     &contentType,
+			ContentEncoding: &contentEncoding,
 		})
 		cancel()
 		if err != nil {
@@ -5139,6 +5241,19 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error {
 			return fmt.Errorf("expected uploaded object metadata to be %v, instead got %v", meta, resp.Metadata)
 		}
 
+		if resp.ContentType == nil {
+			return fmt.Errorf("expected uploaded object content-type to be %v, instead got nil", contentType)
+		}
+		if *resp.ContentType != contentType {
+			return fmt.Errorf("expected uploaded object content-type to be %v, instead got %v", contentType, *resp.ContentType)
+		}
+		if resp.ContentEncoding == nil {
+			return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got nil", contentEncoding)
+		}
+		if *resp.ContentEncoding != contentEncoding {
+			return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got %v", contentEncoding, *resp.ContentEncoding)
+		}
+
 		return nil
 	})
 }