diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 9e905fa..b9a02fa 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -63,6 +63,7 @@ const ( keyBucketLock key = "Bucketlock" keyObjRetention key = "Objectretention" keyObjLegalHold key = "Objectlegalhold" + keyExpires key = "Vgwexpires" onameAttr key = "Objname" onameAttrLower key = "objname" metaTmpMultipartPrefix key = ".sgwtmp" + "/multipart" @@ -76,6 +77,7 @@ func (key) Table() map[string]struct{} { "policy": {}, "bucketlock": {}, "objectretention": {}, + "vgwexpires": {}, "objectlegalhold": {}, "objname": {}, ".sgwtmp/multipart": {}, @@ -292,14 +294,27 @@ func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket strin return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership)) } -func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) { +func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) { tags, err := parseTags(po.Tagging) if err != nil { return s3response.PutObjectOutput{}, err } + metadata := parseMetadata(po.Metadata) + + // Store the "Expires" property in the object metadata + if getString(po.Expires) != "" { + if metadata == nil { + metadata = map[string]*string{ + string(keyExpires): po.Expires, + } + } else { + metadata[string(keyExpires)] = po.Expires + } + } + opts := &blockblob.UploadStreamOptions{ - Metadata: parseMetadata(po.Metadata), + Metadata: metadata, Tags: tags, } @@ -307,6 +322,8 @@ func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respon opts.HTTPHeaders.BlobContentEncoding = po.ContentEncoding opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage opts.HTTPHeaders.BlobContentDisposition = po.ContentDisposition + opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage + opts.HTTPHeaders.BlobCacheControl = po.CacheControl if strings.HasSuffix(*po.Key, "/") { // Hardcode "application/x-directory" for direcoty objects opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DirContentType) @@ -430,17 +447,21 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G } return &s3.GetObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: blobDownloadResponse.ContentLength, - ContentEncoding: blobDownloadResponse.ContentEncoding, - ContentType: contentType, - ETag: (*string)(blobDownloadResponse.ETag), - LastModified: blobDownloadResponse.LastModified, - Metadata: parseAzMetadata(blobDownloadResponse.Metadata), - TagCount: &tagcount, - ContentRange: blobDownloadResponse.ContentRange, - Body: blobDownloadResponse.Body, - StorageClass: types.StorageClassStandard, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: blobDownloadResponse.ContentLength, + ContentEncoding: blobDownloadResponse.ContentEncoding, + ContentType: contentType, + ContentDisposition: blobDownloadResponse.ContentDisposition, + ContentLanguage: blobDownloadResponse.ContentLanguage, + CacheControl: blobDownloadResponse.CacheControl, + ExpiresString: blobDownloadResponse.Metadata[string(keyExpires)], + ETag: (*string)(blobDownloadResponse.ETag), + LastModified: blobDownloadResponse.LastModified, + Metadata: parseAzMetadata(blobDownloadResponse.Metadata), + TagCount: &tagcount, + ContentRange: blobDownloadResponse.ContentRange, + Body: blobDownloadResponse.Body, + StorageClass: types.StorageClassStandard, }, nil } @@ -494,10 +515,11 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3 ContentEncoding: resp.ContentEncoding, ContentLanguage: resp.ContentLanguage, ContentDisposition: resp.ContentDisposition, + CacheControl: resp.CacheControl, + ExpiresString: resp.Metadata[string(keyExpires)], ETag: (*string)(resp.ETag), LastModified: resp.LastModified, Metadata: parseAzMetadata(resp.Metadata), - Expires: resp.ExpiresOn, StorageClass: types.StorageClassStandard, } @@ -826,7 +848,7 @@ func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object string) return nil } -func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { +func (az *Azure) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { if input.ObjectLockLegalHoldStatus != "" || input.ObjectLockMode != "" { bucketLock, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyBucketLock)) if err != nil { @@ -850,6 +872,10 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult meta := parseMetadata(input.Metadata) meta[string(onameAttr)] = input.Key + if getString(input.Expires) != "" { + meta[string(keyExpires)] = input.Expires + } + // parse object tags tagsStr := getString(input.Tagging) tags := map[string]string{} @@ -892,12 +918,13 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult opts := &blockblob.UploadBufferOptions{ Metadata: meta, Tags: tags, - } - if getString(input.ContentType) != "" { - opts.HTTPHeaders = &blob.HTTPHeaders{ - BlobContentType: input.ContentType, - BlobContentEncoding: input.ContentEncoding, - } + HTTPHeaders: &blob.HTTPHeaders{ + BlobContentType: input.ContentType, + BlobContentEncoding: input.ContentEncoding, + BlobCacheControl: input.CacheControl, + BlobContentDisposition: input.ContentDisposition, + BlobContentLanguage: input.ContentLanguage, + }, } // Create and empty blob in .sgwtmp/multipart// @@ -1260,8 +1287,11 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete Tags: parseAzTags(tags.BlobTagSet), } opts.HTTPHeaders = &blob.HTTPHeaders{ - BlobContentType: props.ContentType, - BlobContentEncoding: props.ContentEncoding, + BlobContentType: props.ContentType, + BlobContentEncoding: props.ContentEncoding, + BlobContentDisposition: props.ContentDisposition, + BlobContentLanguage: props.ContentLanguage, + BlobCacheControl: props.CacheControl, } resp, err := client.CommitBlockList(ctx, blockIds, opts) diff --git a/backend/backend.go b/backend/backend.go index 2f4e67a..3af720a 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -48,7 +48,7 @@ type Backend interface { DeleteBucketOwnershipControls(_ context.Context, bucket string) error // multipart operations - CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) + CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) @@ -57,7 +57,7 @@ type Backend interface { UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) // standard object operations - PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) + PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) @@ -151,7 +151,7 @@ func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucke return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { +func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { @@ -173,7 +173,7 @@ func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInpu return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { +func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) { return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 2dbcfac..8126e66 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -85,6 +85,10 @@ const ( metaHdr = "X-Amz-Meta" contentTypeHdr = "content-type" contentEncHdr = "content-encoding" + contentLangHdr = "content-language" + contentDispHdr = "content-disposition" + cacheCtrlHdr = "cache-control" + expiresHdr = "expires" emptyMD5 = "d41d8cd98f00b204e9800998ecf8427e" aclkey = "acl" ownershipkey = "ownership" @@ -1168,7 +1172,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } } -func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { +func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { if mpu.Bucket == nil { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) } @@ -1260,30 +1264,19 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa } } - // set content-type - ctype := getString(mpu.ContentType) - if ctype != "" { - err := p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), - contentTypeHdr, []byte(*mpu.ContentType)) - if err != nil { - // cleanup object if returning error - os.RemoveAll(filepath.Join(tmppath, uploadID)) - os.Remove(tmppath) - return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("set content-type: %w", err) - } - } - - // set content-encoding - cenc := getString(mpu.ContentEncoding) - if cenc != "" { - err := p.meta.StoreAttribute(nil, bucket, filepath.Join(objdir, uploadID), contentEncHdr, - []byte(*mpu.ContentEncoding)) - if err != nil { - // cleanup object if returning error - os.RemoveAll(filepath.Join(tmppath, uploadID)) - os.Remove(tmppath) - return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("set content-encoding: %w", err) - } + err = p.storeObjectMetadata(nil, bucket, filepath.Join(objdir, uploadID), ObjectMetadata{ + ContentType: mpu.ContentType, + ContentEncoding: mpu.ContentEncoding, + ContentDisposition: mpu.ContentDisposition, + ContentLanguage: mpu.ContentLanguage, + CacheControl: mpu.CacheControl, + Expires: mpu.Expires, + }) + if err != nil { + // cleanup object if returning error + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + return s3response.InitiateMultipartUploadResult{}, err } // set object legal hold @@ -1544,9 +1537,14 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } } - userMetaData := make(map[string]string) upiddir := filepath.Join(objdir, uploadID) - cType, cEnc := p.loadUserMetaData(bucket, upiddir, userMetaData) + + userMetaData := make(map[string]string) + objMeta := p.loadObjectMetaData(bucket, upiddir, userMetaData) + err = p.storeObjectMetadata(f.File(), bucket, object, objMeta) + if err != nil { + return nil, err + } objname := filepath.Join(bucket, object) dir := filepath.Dir(objname) @@ -1604,22 +1602,6 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } } - // set content-type - if cType != "" { - err := p.meta.StoreAttribute(f.File(), bucket, object, contentTypeHdr, []byte(cType)) - if err != nil { - return nil, fmt.Errorf("set object content type: %w", err) - } - } - - // set content-encoding - if cEnc != "" { - err := p.meta.StoreAttribute(f.File(), bucket, object, contentEncHdr, []byte(cEnc)) - if err != nil { - return nil, fmt.Errorf("set object content encoding: %w", err) - } - } - // load and set legal hold lHold, err := p.meta.RetrieveAttribute(nil, bucket, upiddir, objectLegalHoldKey) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -1840,46 +1822,118 @@ func (p *Posix) retrieveUploadId(bucket, object string) (string, [32]byte, error return entries[0].Name(), sum, nil } -// fll out the user metadata map with the metadata for the object -// and return the content type and encoding -func (p *Posix) loadUserMetaData(bucket, object string, m map[string]string) (string, string) { +type ObjectMetadata struct { + ContentType *string + ContentEncoding *string + ContentDisposition *string + ContentLanguage *string + CacheControl *string + Expires *string +} + +// fill out the user metadata map with the metadata for the object +// and return object meta properties as `ObjectMetadata` +func (p *Posix) loadObjectMetaData(bucket, object string, m map[string]string) ObjectMetadata { ents, err := p.meta.ListAttributes(bucket, object) if err != nil || len(ents) == 0 { - return "", "" + return ObjectMetadata{} } - for _, e := range ents { - if !isValidMeta(e) { - continue + + if m != nil { + for _, e := range ents { + if !isValidMeta(e) { + continue + } + b, err := p.meta.RetrieveAttribute(nil, bucket, object, e) + if err != nil { + continue + } + if b == nil { + m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = "" + continue + } + m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b) } - b, err := p.meta.RetrieveAttribute(nil, bucket, object, e) + } + + var result ObjectMetadata + + b, err := p.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr) + if err == nil { + result.ContentType = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr) + if err == nil { + result.ContentEncoding = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(nil, bucket, object, contentDispHdr) + if err == nil { + result.ContentDisposition = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(nil, bucket, object, contentLangHdr) + if err == nil { + result.ContentLanguage = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(nil, bucket, object, cacheCtrlHdr) + if err == nil { + result.CacheControl = backend.GetPtrFromString(string(b)) + } + + b, err = p.meta.RetrieveAttribute(nil, bucket, object, expiresHdr) + if err == nil { + result.Expires = backend.GetPtrFromString(string(b)) + } + + return result +} + +func (p *Posix) storeObjectMetadata(f *os.File, bucket, object string, m ObjectMetadata) error { + if getString(m.ContentType) != "" { + err := p.meta.StoreAttribute(f, bucket, object, contentTypeHdr, []byte(*m.ContentType)) if err != nil { - continue + return fmt.Errorf("set content-type: %w", err) } - if b == nil { - m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = "" - continue + } + if getString(m.ContentEncoding) != "" { + err := p.meta.StoreAttribute(f, bucket, object, contentEncHdr, []byte(*m.ContentEncoding)) + if err != nil { + return fmt.Errorf("set content-encoding: %w", err) + } + } + if getString(m.ContentDisposition) != "" { + err := p.meta.StoreAttribute(f, bucket, object, contentDispHdr, []byte(*m.ContentDisposition)) + if err != nil { + return fmt.Errorf("set content-disposition: %w", err) + } + } + if getString(m.ContentLanguage) != "" { + err := p.meta.StoreAttribute(f, bucket, object, contentLangHdr, []byte(*m.ContentLanguage)) + if err != nil { + return fmt.Errorf("set content-language: %w", err) + } + } + if getString(m.CacheControl) != "" { + err := p.meta.StoreAttribute(f, bucket, object, cacheCtrlHdr, []byte(*m.CacheControl)) + if err != nil { + return fmt.Errorf("set cache-control: %w", err) + } + } + if getString(m.Expires) != "" { + err := p.meta.StoreAttribute(f, bucket, object, expiresHdr, []byte(*m.Expires)) + if err != nil { + return fmt.Errorf("set cache-control: %w", err) } - m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b) } - var contentType, contentEncoding string - b, _ := p.meta.RetrieveAttribute(nil, bucket, object, contentTypeHdr) - contentType = string(b) - - b, _ = p.meta.RetrieveAttribute(nil, bucket, object, contentEncHdr) - contentEncoding = string(b) - - return contentType, contentEncoding + return nil } func isValidMeta(val string) bool { - if strings.HasPrefix(val, metaHdr) { - return true - } - if strings.EqualFold(val, "Expires") { - return true - } - return false + return strings.HasPrefix(val, metaHdr) } func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error { @@ -2198,7 +2252,7 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp userMetaData := make(map[string]string) upiddir := filepath.Join(objdir, uploadID) - p.loadUserMetaData(bucket, upiddir, userMetaData) + p.loadObjectMetaData(bucket, upiddir, userMetaData) return s3response.ListPartsResult{ Bucket: bucket, @@ -2606,7 +2660,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) }, nil } -func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) { +func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} @@ -2683,6 +2737,13 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) } + // set "application/x-directory" content-type + err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, contentTypeHdr, + []byte(backend.DirContentType)) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err) + } + // for directory object no version is created return s3response.PutObjectOutput{ ETag: emptyMD5, @@ -2853,22 +2914,16 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) } - ctype := getString(po.ContentType) - if ctype != "" { - err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, contentTypeHdr, - []byte(*po.ContentType)) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err) - } - } - - cenc := getString(po.ContentEncoding) - if cenc != "" { - err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, contentEncHdr, - []byte(*po.ContentEncoding)) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("set content-encoding attr: %w", err) - } + err = p.storeObjectMetadata(f.File(), *po.Bucket, *po.Key, ObjectMetadata{ + ContentType: po.ContentType, + ContentEncoding: po.ContentEncoding, + ContentLanguage: po.ContentLanguage, + ContentDisposition: po.ContentDisposition, + CacheControl: po.CacheControl, + Expires: po.Expires, + }) + if err != nil { + return s3response.PutObjectOutput{}, err } if versionID != "" && versionID != nullVersionId { @@ -3392,9 +3447,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO if fi.IsDir() { userMetaData := make(map[string]string) - _, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData) - contentType := backend.DirContentType - + objMeta := p.loadObjectMetaData(bucket, object, userMetaData) b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) etag := string(b) if err != nil { @@ -3412,17 +3465,21 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO } return &s3.GetObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: &length, - ContentEncoding: &contentEncoding, - ContentType: &contentType, - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), - Metadata: userMetaData, - TagCount: tagCount, - ContentRange: &contentRange, - StorageClass: types.StorageClassStandard, - VersionId: &versionId, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + ContentEncoding: objMeta.ContentEncoding, + ContentType: objMeta.ContentType, + ContentLanguage: objMeta.ContentLanguage, + ContentDisposition: objMeta.ContentDisposition, + CacheControl: objMeta.CacheControl, + ExpiresString: objMeta.Expires, + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + Metadata: userMetaData, + TagCount: tagCount, + ContentRange: &contentRange, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, }, nil } @@ -3440,7 +3497,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO userMetaData := make(map[string]string) - contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData) + objMeta := p.loadObjectMetaData(bucket, object, userMetaData) b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) etag := string(b) @@ -3487,24 +3544,28 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO } return &s3.GetObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: &length, - ContentEncoding: &contentEncoding, - ContentType: &contentType, - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), - Metadata: userMetaData, - TagCount: tagCount, - ContentRange: &contentRange, - StorageClass: types.StorageClassStandard, - VersionId: &versionId, - Body: body, - ChecksumCRC32: checksums.CRC32, - ChecksumCRC32C: checksums.CRC32C, - ChecksumSHA1: checksums.SHA1, - ChecksumSHA256: checksums.SHA256, - ChecksumCRC64NVME: checksums.CRC64NVME, - ChecksumType: cType, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + ContentEncoding: objMeta.ContentEncoding, + ContentType: objMeta.ContentType, + ContentDisposition: objMeta.ContentDisposition, + ContentLanguage: objMeta.ContentLanguage, + CacheControl: objMeta.CacheControl, + ExpiresString: objMeta.Expires, + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + Metadata: userMetaData, + TagCount: tagCount, + ContentRange: &contentRange, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, + Body: body, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumType: cType, }, nil } @@ -3647,11 +3708,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } userMetaData := make(map[string]string) - contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData) - - if fi.IsDir() { - contentType = backend.DirContentType - } + objMeta := p.loadObjectMetaData(bucket, object, userMetaData) b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) etag := string(b) @@ -3696,8 +3753,12 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return &s3.HeadObjectOutput{ ContentLength: &size, - ContentType: &contentType, - ContentEncoding: &contentEncoding, + ContentType: objMeta.ContentType, + ContentEncoding: objMeta.ContentEncoding, + ContentDisposition: objMeta.ContentDisposition, + ContentLanguage: objMeta.ContentLanguage, + CacheControl: objMeta.CacheControl, + ExpiresString: objMeta.Expires, ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: userMetaData, @@ -3840,7 +3901,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. } mdmap := make(map[string]string) - p.loadUserMetaData(srcBucket, srcObject, mdmap) + p.loadObjectMetaData(srcBucket, srcObject, mdmap) var etag string var version *string @@ -3953,7 +4014,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. } res, err := p.PutObject(ctx, - &s3.PutObjectInput{ + s3response.PutObjectInput{ Bucket: &dstBucket, Key: &dstObject, Body: f, diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index 718f40e..0fedcee 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -254,7 +254,7 @@ func (s *S3Proxy) ListObjectVersions(ctx context.Context, input *s3.ListObjectVe var defTime = time.Time{} -func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { +func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { if input.CacheControl != nil && *input.CacheControl == "" { input.CacheControl = nil } @@ -270,7 +270,7 @@ func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMul if input.ContentType != nil && *input.ContentType == "" { input.ContentType = nil } - if input.Expires != nil && *input.Expires == defTime { + if input.Expires != nil && *input.Expires == "" { input.Expires = nil } if input.GrantFullControl != nil && *input.GrantFullControl == "" { @@ -313,7 +313,47 @@ func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMul input.WebsiteRedirectLocation = nil } - out, err := s.client.CreateMultipartUpload(ctx, input) + var expires *time.Time + if input.Expires != nil { + exp, err := time.Parse(time.RFC1123, *input.Expires) + if err == nil { + expires = &exp + } + } + + out, err := s.client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: input.Bucket, + Key: input.Key, + ExpectedBucketOwner: input.ExpectedBucketOwner, + CacheControl: input.CacheControl, + ContentDisposition: input.ContentDisposition, + ContentEncoding: input.ContentEncoding, + ContentLanguage: input.ContentLanguage, + ContentType: input.ContentType, + Expires: expires, + SSECustomerAlgorithm: input.SSECustomerAlgorithm, + SSECustomerKey: input.SSECustomerKey, + SSECustomerKeyMD5: input.SSECustomerKeyMD5, + SSEKMSEncryptionContext: input.SSEKMSEncryptionContext, + SSEKMSKeyId: input.SSEKMSKeyId, + GrantFullControl: input.GrantFullControl, + GrantRead: input.GrantRead, + GrantReadACP: input.GrantReadACP, + GrantWriteACP: input.GrantWriteACP, + Tagging: input.Tagging, + WebsiteRedirectLocation: input.WebsiteRedirectLocation, + BucketKeyEnabled: input.BucketKeyEnabled, + ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate, + Metadata: input.Metadata, + ACL: input.ACL, + ChecksumAlgorithm: input.ChecksumAlgorithm, + ChecksumType: input.ChecksumType, + ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus, + ObjectLockMode: input.ObjectLockMode, + RequestPayer: input.RequestPayer, + ServerSideEncryption: input.ServerSideEncryption, + StorageClass: input.StorageClass, + }) if err != nil { return s3response.InitiateMultipartUploadResult{}, handleError(err) } @@ -617,7 +657,7 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn }, nil } -func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3response.PutObjectOutput, error) { +func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput) (s3response.PutObjectOutput, error) { if input.CacheControl != nil && *input.CacheControl == "" { input.CacheControl = nil } @@ -654,7 +694,7 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3re if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" { input.ExpectedBucketOwner = nil } - if input.Expires != nil && *input.Expires == defTime { + if input.Expires != nil && *input.Expires == "" { input.Expires = nil } if input.GrantFullControl != nil && *input.GrantFullControl == "" { @@ -702,9 +742,53 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3re input.ObjectLockMode = "" input.ObjectLockLegalHoldStatus = "" + var expire *time.Time + if input.Expires != nil { + exp, err := time.Parse(time.RFC1123, *input.Expires) + if err == nil { + expire = &exp + } + } + // streaming backend is not seekable, // use unsigned payload for streaming ops - output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions( + output, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: input.Bucket, + Key: input.Key, + ContentLength: input.ContentLength, + ContentType: input.ContentType, + ContentEncoding: input.ContentEncoding, + ContentDisposition: input.ContentDisposition, + ContentLanguage: input.ContentLanguage, + CacheControl: input.CacheControl, + Expires: expire, + Metadata: input.Metadata, + Body: input.Body, + Tagging: input.Tagging, + ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate, + ObjectLockMode: input.ObjectLockMode, + ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus, + ChecksumAlgorithm: input.ChecksumAlgorithm, + ChecksumCRC32: input.ChecksumCRC32, + ChecksumCRC32C: input.ChecksumCRC32C, + ChecksumSHA1: input.ChecksumSHA1, + ChecksumSHA256: input.ChecksumSHA256, + ChecksumCRC64NVME: input.ChecksumCRC64NVME, + ContentMD5: input.ContentMD5, + ExpectedBucketOwner: input.ExpectedBucketOwner, + GrantFullControl: input.GrantFullControl, + GrantRead: input.GrantRead, + GrantReadACP: input.GrantReadACP, + GrantWriteACP: input.GrantWriteACP, + IfMatch: input.IfMatch, + IfNoneMatch: input.IfNoneMatch, + SSECustomerAlgorithm: input.SSECustomerAlgorithm, + SSECustomerKey: input.SSECustomerKey, + SSECustomerKeyMD5: input.SSECustomerKeyMD5, + SSEKMSEncryptionContext: input.SSEKMSEncryptionContext, + SSEKMSKeyId: input.SSEKMSKeyId, + WebsiteRedirectLocation: input.WebsiteRedirectLocation, + }, s3.WithAPIOptions( v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware, )) if err != nil { diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index b29b281..7f22c6d 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -38,7 +38,7 @@ var _ backend.Backend = &BackendMock{} // CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error { // panic("mock out the CreateBucket method") // }, -// CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { +// CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { // panic("mock out the CreateMultipartUpload method") // }, // DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error { @@ -140,7 +140,7 @@ var _ backend.Backend = &BackendMock{} // PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { // panic("mock out the PutBucketVersioning method") // }, -// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) { +// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) { // panic("mock out the PutObject method") // }, // PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error { @@ -199,7 +199,7 @@ type BackendMock struct { CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error // CreateMultipartUploadFunc mocks the CreateMultipartUpload method. - CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) + CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) // DeleteBucketFunc mocks the DeleteBucket method. DeleteBucketFunc func(contextMoqParam context.Context, bucket string) error @@ -301,7 +301,7 @@ type BackendMock struct { PutBucketVersioningFunc func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error // PutObjectFunc mocks the PutObject method. - PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) + PutObjectFunc func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) // PutObjectAclFunc mocks the PutObjectAcl method. PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error @@ -382,7 +382,7 @@ type BackendMock struct { // ContextMoqParam is the contextMoqParam argument value. ContextMoqParam context.Context // CreateMultipartUploadInput is the createMultipartUploadInput argument value. - CreateMultipartUploadInput *s3.CreateMultipartUploadInput + CreateMultipartUploadInput s3response.CreateMultipartUploadInput } // DeleteBucket holds details about calls to the DeleteBucket method. DeleteBucket []struct { @@ -640,7 +640,7 @@ type BackendMock struct { // ContextMoqParam is the contextMoqParam argument value. ContextMoqParam context.Context // PutObjectInput is the putObjectInput argument value. - PutObjectInput *s3.PutObjectInput + PutObjectInput s3response.PutObjectInput } // PutObjectAcl holds details about calls to the PutObjectAcl method. PutObjectAcl []struct { @@ -974,13 +974,13 @@ func (mock *BackendMock) CreateBucketCalls() []struct { } // CreateMultipartUpload calls CreateMultipartUploadFunc. -func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { +func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { if mock.CreateMultipartUploadFunc == nil { panic("BackendMock.CreateMultipartUploadFunc: method is nil but Backend.CreateMultipartUpload was just called") } callInfo := struct { ContextMoqParam context.Context - CreateMultipartUploadInput *s3.CreateMultipartUploadInput + CreateMultipartUploadInput s3response.CreateMultipartUploadInput }{ ContextMoqParam: contextMoqParam, CreateMultipartUploadInput: createMultipartUploadInput, @@ -997,11 +997,11 @@ func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, // len(mockedBackend.CreateMultipartUploadCalls()) func (mock *BackendMock) CreateMultipartUploadCalls() []struct { ContextMoqParam context.Context - CreateMultipartUploadInput *s3.CreateMultipartUploadInput + CreateMultipartUploadInput s3response.CreateMultipartUploadInput } { var calls []struct { ContextMoqParam context.Context - CreateMultipartUploadInput *s3.CreateMultipartUploadInput + CreateMultipartUploadInput s3response.CreateMultipartUploadInput } mock.lockCreateMultipartUpload.RLock() calls = mock.calls.CreateMultipartUpload @@ -2238,13 +2238,13 @@ func (mock *BackendMock) PutBucketVersioningCalls() []struct { } // PutObject calls PutObjectFunc. -func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (s3response.PutObjectOutput, error) { +func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) { if mock.PutObjectFunc == nil { panic("BackendMock.PutObjectFunc: method is nil but Backend.PutObject was just called") } callInfo := struct { ContextMoqParam context.Context - PutObjectInput *s3.PutObjectInput + PutObjectInput s3response.PutObjectInput }{ ContextMoqParam: contextMoqParam, PutObjectInput: putObjectInput, @@ -2261,11 +2261,11 @@ func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInp // len(mockedBackend.PutObjectCalls()) func (mock *BackendMock) PutObjectCalls() []struct { ContextMoqParam context.Context - PutObjectInput *s3.PutObjectInput + PutObjectInput s3response.PutObjectInput } { var calls []struct { ContextMoqParam context.Context - PutObjectInput *s3.PutObjectInput + PutObjectInput s3response.PutObjectInput } mock.lockPutObject.RLock() calls = mock.calls.PutObject diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 257eeba..75c9171 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -556,7 +556,36 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Value: acceptRanges, }, } - + if getstring(res.ContentDisposition) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "Content-Disposition", + Value: getstring(res.ContentDisposition), + }) + } + if getstring(res.ContentEncoding) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "Content-Encoding", + Value: getstring(res.ContentEncoding), + }) + } + if getstring(res.ContentLanguage) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "Content-Language", + Value: getstring(res.ContentLanguage), + }) + } + if getstring(res.CacheControl) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "Cache-Control", + Value: getstring(res.CacheControl), + }) + } + if getstring(res.ExpiresString) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "Expires", + Value: getstring(res.ExpiresString), + }) + } if getstring(res.ContentRange) != "" { hdrs = append(hdrs, utils.CustomHeader{ Key: "Content-Range", @@ -569,12 +598,6 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Value: res.LastModified.Format(timefmt), }) } - if getstring(res.ContentEncoding) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Content-Encoding", - Value: getstring(res.ContentEncoding), - }) - } if res.TagCount != nil { hdrs = append(hdrs, utils.CustomHeader{ Key: "x-amz-tagging-count", @@ -1705,6 +1728,9 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { isRoot := ctx.Locals("isRoot").(bool) contentType := ctx.Get("Content-Type") contentEncoding := ctx.Get("Content-Encoding") + contentDisposition := ctx.Get("Content-Disposition") + contentLanguage := ctx.Get("Content-Language") + cacheControl := ctx.Get("Cache-Control") parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) tagging := ctx.Get("x-amz-tagging") @@ -2500,6 +2526,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { }) } + expires := ctx.Get("Expires") + var body io.Reader bodyi := ctx.Locals("body-reader") if bodyi != nil { @@ -2510,12 +2538,16 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ctx.Locals("logReqBody", false) res, err := c.be.PutObject(ctx.Context(), - &s3.PutObjectInput{ + s3response.PutObjectInput{ Bucket: &bucket, Key: &keyStart, ContentLength: &contentLength, ContentType: &contentType, ContentEncoding: &contentEncoding, + ContentDisposition: &contentDisposition, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, Metadata: metadata, Body: body, Tagging: &tagging, @@ -3164,6 +3196,36 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: getstring(res.Restore), }, } + if getstring(res.ContentDisposition) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "Content-Disposition", + Value: getstring(res.ContentDisposition), + }) + } + if getstring(res.ContentEncoding) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "Content-Encoding", + Value: getstring(res.ContentEncoding), + }) + } + if getstring(res.ContentLanguage) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "Content-Language", + Value: getstring(res.ContentLanguage), + }) + } + if getstring(res.CacheControl) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "Cache-Control", + Value: getstring(res.CacheControl), + }) + } + if getstring(res.ExpiresString) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "Expires", + Value: getstring(res.ExpiresString), + }) + } if res.ObjectLockMode != "" { headers = append(headers, utils.CustomHeader{ Key: "x-amz-object-lock-mode", @@ -3196,12 +3258,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: lastmod, }) } - if res.ContentEncoding != nil { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Encoding", - Value: getstring(res.ContentEncoding), - }) - } if res.StorageClass != "" { headers = append(headers, utils.CustomHeader{ Key: "x-amz-storage-class", @@ -3278,6 +3334,9 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { isRoot := ctx.Locals("isRoot").(bool) parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) contentType := ctx.Get("Content-Type") + contentDisposition := ctx.Get("Content-Disposition") + contentLanguage := ctx.Get("Content-Language") + cacheControl := ctx.Get("Cache-Control") contentEncoding := ctx.Get("Content-Encoding") tagging := ctx.Get("X-Amz-Tagging") @@ -3604,13 +3663,19 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { }) } + expires := ctx.Get("Expires") + res, err := c.be.CreateMultipartUpload(ctx.Context(), - &s3.CreateMultipartUploadInput{ + s3response.CreateMultipartUploadInput{ Bucket: &bucket, Key: &key, Tagging: &tagging, ContentType: &contentType, ContentEncoding: &contentEncoding, + ContentDisposition: &contentDisposition, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, ObjectLockRetainUntilDate: &objLockState.RetainUntilDate, ObjectLockMode: objLockState.ObjectLockMode, ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus, diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 14a5be6..3be41f7 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -980,7 +980,7 @@ func TestS3ApiController_PutActions(t *testing.T) { CopyObjectResult: &types.CopyObjectResult{}, }, nil }, - PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) { + PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) { return s3response.PutObjectOutput{}, nil }, UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) { @@ -1769,7 +1769,7 @@ func TestS3ApiController_CreateActions(t *testing.T) { CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { return &s3.CompleteMultipartUploadOutput{}, nil }, - CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { + CreateMultipartUploadFunc: func(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { return s3response.InitiateMultipartUploadResult{}, nil }, SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) { diff --git a/s3response/s3response.go b/s3response/s3response.go index 1660392..58588d2 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -16,6 +16,7 @@ package s3response import ( "encoding/xml" + "io" "time" "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -447,6 +448,82 @@ type PutObjectRetentionInput struct { RetainUntilDate AmzDate } +type PutObjectInput struct { + ContentLength *int64 + ObjectLockRetainUntilDate *time.Time + + Bucket *string + Key *string + ContentType *string + ContentEncoding *string + ContentDisposition *string + ContentLanguage *string + CacheControl *string + Expires *string + Tagging *string + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string + ChecksumCRC64NVME *string + ContentMD5 *string + ExpectedBucketOwner *string + GrantFullControl *string + GrantRead *string + GrantReadACP *string + GrantWriteACP *string + IfMatch *string + IfNoneMatch *string + SSECustomerAlgorithm *string + SSECustomerKey *string + SSECustomerKeyMD5 *string + SSEKMSEncryptionContext *string + SSEKMSKeyId *string + WebsiteRedirectLocation *string + + ObjectLockMode types.ObjectLockMode + ObjectLockLegalHoldStatus types.ObjectLockLegalHoldStatus + ChecksumAlgorithm types.ChecksumAlgorithm + + Metadata map[string]string + Body io.Reader +} + +type CreateMultipartUploadInput struct { + Bucket *string + Key *string + ExpectedBucketOwner *string + CacheControl *string + ContentDisposition *string + ContentEncoding *string + ContentLanguage *string + ContentType *string + Expires *string + SSECustomerAlgorithm *string + SSECustomerKey *string + SSECustomerKeyMD5 *string + SSEKMSEncryptionContext *string + SSEKMSKeyId *string + GrantFullControl *string + GrantRead *string + GrantReadACP *string + GrantWriteACP *string + Tagging *string + WebsiteRedirectLocation *string + BucketKeyEnabled *bool + ObjectLockRetainUntilDate *time.Time + Metadata map[string]string + + ACL types.ObjectCannedACL + ChecksumAlgorithm types.ChecksumAlgorithm + ChecksumType types.ChecksumType + ObjectLockLegalHoldStatus types.ObjectLockLegalHoldStatus + ObjectLockMode types.ObjectLockMode + RequestPayer types.RequestPayer + ServerSideEncryption types.ServerSideEncryption + StorageClass types.StorageClass +} + type AmzDate struct { time.Time } diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 6d55480..5cc00b5 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -304,7 +304,6 @@ func TestCreateMultipartUpload(s *S3Conf) { CreateMultipartUpload_with_metadata(s) CreateMultipartUpload_with_invalid_tagging(s) CreateMultipartUpload_with_tagging(s) - CreateMultipartUpload_with_content_type(s) CreateMultipartUpload_with_object_lock(s) CreateMultipartUpload_with_object_lock_not_enabled(s) CreateMultipartUpload_with_object_lock_invalid_retention(s) @@ -933,7 +932,6 @@ func GetIntTests() IntTests { "CreateMultipartUpload_with_metadata": CreateMultipartUpload_with_metadata, "CreateMultipartUpload_with_invalid_tagging": CreateMultipartUpload_with_invalid_tagging, "CreateMultipartUpload_with_tagging": CreateMultipartUpload_with_tagging, - "CreateMultipartUpload_with_content_type": CreateMultipartUpload_with_content_type, "CreateMultipartUpload_with_object_lock": CreateMultipartUpload_with_object_lock, "CreateMultipartUpload_with_object_lock_not_enabled": CreateMultipartUpload_with_object_lock_not_enabled, "CreateMultipartUpload_with_object_lock_invalid_retention": CreateMultipartUpload_with_object_lock_invalid_retention, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 44d9e53..be3169c 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -41,6 +41,7 @@ var ( shortTimeout = 10 * time.Second longTimeout = 60 * time.Second iso8601Format = "20060102T150405Z" + timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" nullVersionId = "null" ) @@ -3594,13 +3595,19 @@ func HeadObject_success(s *S3Conf) error { "key1": "val1", "key2": "val2", } - ctype := defaultContentType + ctype, cDisp, cEnc, cLang := defaultContentType, "cont-desp", "json", "eng" + cacheControl, expires := "cache-ctrl", time.Now().Add(time.Hour*2) _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - Metadata: meta, - ContentType: &ctype, + Bucket: &bucket, + Key: &obj, + Metadata: meta, + ContentType: &ctype, + ContentDisposition: &cDisp, + ContentEncoding: &cEnc, + ContentLanguage: &cLang, + CacheControl: &cacheControl, + Expires: &expires, }, s3client) if err != nil { return err @@ -3627,7 +3634,22 @@ func HeadObject_success(s *S3Conf) error { return fmt.Errorf("expected data length %v, instead got %v", dataLen, contentLength) } if *out.ContentType != defaultContentType { - return fmt.Errorf("expected content type %v, instead got %v", defaultContentType, *out.ContentType) + return fmt.Errorf("expected Content-Type %v, instead got %v", defaultContentType, *out.ContentType) + } + if getString(out.ContentDisposition) != cDisp { + return fmt.Errorf("expected Content-Disposition %v, instead got %v", cDisp, getString(out.ContentDisposition)) + } + if getString(out.ContentEncoding) != cEnc { + return fmt.Errorf("expected Content-Encoding %v, instead got %v", cEnc, getString(out.ContentEncoding)) + } + if getString(out.ContentLanguage) != cLang { + return fmt.Errorf("expected Content-Language %v, instead got %v", cLang, getString(out.ContentLanguage)) + } + if getString(out.ExpiresString) != expires.UTC().Format(timefmt) { + return fmt.Errorf("expected Expiress %v, instead got %v", expires.UTC().Format(timefmt), getString(out.ExpiresString)) + } + if getString(out.CacheControl) != cacheControl { + return fmt.Errorf("expected Cache-Control %v, instead got %v", cacheControl, getString(out.CacheControl)) } if out.StorageClass != types.StorageClassStandard { return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) @@ -4268,12 +4290,18 @@ func GetObject_success(s *S3Conf) error { testName := "GetObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { dataLength, obj := int64(1234567), "my-obj" - ctype := defaultContentType + ctype, cDisp, cEnc, cLang := defaultContentType, "cont-desp", "json", "eng" + cacheControl, expires := "cache-ctrl", time.Now().Add(time.Hour*2) r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - ContentType: &ctype, + Bucket: &bucket, + Key: &obj, + ContentType: &ctype, + ContentDisposition: &cDisp, + ContentEncoding: &cEnc, + ContentLanguage: &cLang, + Expires: &expires, + CacheControl: &cacheControl, }, s3client) if err != nil { return err @@ -4291,8 +4319,23 @@ func GetObject_success(s *S3Conf) error { if *out.ContentLength != dataLength { return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) } - if *out.ContentType != defaultContentType { - return fmt.Errorf("expected content type %v, instead got %v", defaultContentType, *out.ContentType) + if getString(out.ContentType) != defaultContentType { + return fmt.Errorf("expected Content-Type %v, instead got %v", defaultContentType, getString(out.ContentType)) + } + if getString(out.ContentDisposition) != cDisp { + return fmt.Errorf("expected Content-Disposition %v, instead got %v", cDisp, getString(out.ContentDisposition)) + } + if getString(out.ContentEncoding) != cEnc { + return fmt.Errorf("expected Content-Encoding %v, instead got %v", cEnc, getString(out.ContentEncoding)) + } + if getString(out.ContentLanguage) != cLang { + return fmt.Errorf("expected Content-Language %v, instead got %v", cLang, getString(out.ContentLanguage)) + } + if getString(out.ExpiresString) != expires.UTC().Format(timefmt) { + return fmt.Errorf("expected Expiress %v, instead got %v", expires.UTC().Format(timefmt), getString(out.ExpiresString)) + } + if getString(out.CacheControl) != cacheControl { + return fmt.Errorf("expected Cache-Control %v, instead got %v", cacheControl, getString(out.CacheControl)) } if out.StorageClass != types.StorageClassStandard { return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) @@ -6610,16 +6653,20 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error { "prop1": "val1", "prop2": "val2", } - contentType := "application/text" - contentEncoding := "testenc" + cType, cEnc, cDesp, cLang := "application/text", "testenc", "testdesp", "sp" + cacheControl, expires := "no-cache", time.Now().Add(time.Hour*5) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - Metadata: meta, - ContentType: &contentType, - ContentEncoding: &contentEncoding, + Bucket: &bucket, + Key: &obj, + Metadata: meta, + ContentType: &cType, + ContentEncoding: &cEnc, + ContentDisposition: &cDesp, + ContentLanguage: &cLang, + CacheControl: &cacheControl, + Expires: &expires, }) cancel() if err != nil { @@ -6667,78 +6714,23 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error { return fmt.Errorf("expected uploaded object metadata to be %v, instead got %v", meta, resp.Metadata) } - if resp.ContentType == nil { - return fmt.Errorf("expected uploaded object content-type to be %v, instead got nil", contentType) + if getString(resp.ContentType) != cType { + return fmt.Errorf("expected uploaded object content-type to be %v, instead got %v", cType, getString(resp.ContentType)) } - if *resp.ContentType != contentType { - return fmt.Errorf("expected uploaded object content-type to be %v, instead got %v", contentType, *resp.ContentType) + if getString(resp.ContentEncoding) != cEnc { + return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got %v", cEnc, getString(resp.ContentEncoding)) } - if resp.ContentEncoding == nil { - return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got nil", contentEncoding) + if getString(resp.ContentLanguage) != cLang { + return fmt.Errorf("expected uploaded object content-language to be %v, instead got %v", cLang, getString(resp.ContentLanguage)) } - if *resp.ContentEncoding != contentEncoding { - return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got %v", contentEncoding, *resp.ContentEncoding) + if getString(resp.ContentDisposition) != cDesp { + return fmt.Errorf("expected uploaded object content-disposition to be %v, instead got %v", cDesp, getString(resp.ContentDisposition)) } - - return nil - }) -} - -func CreateMultipartUpload_with_content_type(s *S3Conf) error { - testName := "CreateMultipartUpload_with_content_type" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - cType := "application/octet-stream" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - ContentType: &cType, - }) - cancel() - if err != nil { - return err + if getString(resp.CacheControl) != cacheControl { + return fmt.Errorf("expected uploaded object cache-control to be %v, instead got %v", cacheControl, getString(resp.CacheControl)) } - - parts, _, err := uploadParts(s3client, 100, 1, bucket, obj, *out.UploadId) - if err != nil { - return err - } - - compParts := []types.CompletedPart{} - for _, el := range parts { - compParts = append(compParts, types.CompletedPart{ - ETag: el.ETag, - PartNumber: el.PartNumber, - }) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: out.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: compParts, - }, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - resp, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } - - if *resp.ContentType != cType { - return fmt.Errorf("expected uploaded object content-type to be %v, instead got %v", cType, *resp.ContentType) + if getString(resp.ExpiresString) != expires.UTC().Format(timefmt) { + return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got %v", expires.UTC().Format(timefmt), getString(resp.ExpiresString)) } return nil