mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-24 00:44:23 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
fce9d9bf69dad72c09abc54cb45fdcfe1ffabc44
versitygw/backend/azure/azure.go
T
Nils LegerandGitHub c638503b90 feat(azure): carry Azure blob marker in continuation token for pagination 
* feat(azure): carry Azure blob marker in continuation token for pagination

Azure blob markers are opaque values only Azure may mint, so an S3 object
key can never be passed back to Azure as a listing marker. Paging the
underlying Azure listing with an S3 key therefore failed or re-scanned the
whole prefix on every page.

Introduce azMarkerToken, an opaque S3 continuation token that carries the
Azure marker alongside the last returned key: the Azure marker resumes the
blob listing where it stopped, and the last key filters out already-returned
entries. Tokens are versioned with a "vgw1." prefix; anything without it is
treated as a plain key, so tokens from older versions and hand-crafted
markers keep working. The shared listBlobs helper now backs both ListObjects
and ListObjectsV2, applying the S3 marker and delimiter client side.
ListObjectsV2 pages efficiently via the token; ListObjects (v1) has no token
to carry state and walks the prefix from the start each page.

Add unit tests (token round-trip, marker-resumed pagination, delimiter and
common-prefix handling, multipart filtering) driven by a fake Azure
container, and integration tests covering full and delimited pagination.

Signed-off-by: Nils Leger <nils.leger@getflip.com>

* fix: token is now bound to delimiter too

Signed-off-by: Nils Leger <nils.leger@getflip.com>

---------

Signed-off-by: Nils Leger <nils.leger@getflip.com>
2026-08-27 16:40:43 -07:00

2698 lines
80 KiB
Go
Raw Blame History

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package azure
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"net/url"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/uuid"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
// When getting container metadata with GetProperties method the sdk returns
// the first letter capital, when accessing the metadata after listing the containers
// it returns the first letter lower
type key string
const (
keyAclCapital key = "Acl"
keyAclLower key = "acl"
keyOwnership key = "Ownership"
keyTags key = "Tags"
keyPolicy key = "Policy"
keyCors key = "Cors"
keyWebsite key = "Website"
keyBucketLock key = "Bucketlock"
keyObjRetention key = "Objectretention"
keyObjLegalHold key = "Objectlegalhold"
keyExpires key = "Vgwexpires"
keyWebsiteRedirect key = "Vgwwebsiteredirect"
onameAttr key = "Objname"
onameAttrLower key = "objname"
metaTmpMultipartPrefix key = ".sgwtmp" + "/multipart"
// keyMpZeroBytesParts tracks zero-byte upload parts in the sgwtmp metadata.
// Azure StageBlock rejects Content-Length: 0, so zero-byte parts are stored here.
keyMpZeroBytesParts key = "Zerobytesparts"
// keyMpMetadata stores multipart upload part-offset metadata on the final
// committed blob so that GetObject/HeadObject can serve individual parts
// by part-number.
keyMpMetadata key = "Mpmetadata"
defaultListingMaxKeys = 1000
)
func (key) Table() map[string]struct{} {
return map[string]struct{}{
"acl": {},
"ownership": {},
"tags": {},
"policy": {},
"bucketlock": {},
"website": {},
"objectretention": {},
"vgwexpires": {},
"vgwwebsiteredirect": {},
"objectlegalhold": {},
"objname": {},
".sgwtmp/multipart": {},
"mpmetadata": {},
}
}
type Azure struct {
backend.BackendUnsupported
client *azblob.Client
sharedkeyCreds *azblob.SharedKeyCredential
defaultCreds *azidentity.DefaultAzureCredential
serviceURL string
sasToken string
copyObjectThreshold int64
// copySASVersion overrides the service version used to sign the copy-source
// SAS for server-side copy. Empty means use the SDK default. This exists so
// endpoints that lag the SDK's SAS version (e.g. Azurite) can verify the
// signature; production leaves it unset.
copySASVersion string
}
var _ backend.Backend = &Azure{}
func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThreshold int64, copySASVersion string) (*Azure, error) {
url := serviceURL
if serviceURL == "" && accountName != "" {
// if not otherwise specified, use the typical form:
// http(s)://<account>.blob.core.windows.net/
url = fmt.Sprintf("https://%s.blob.core.windows.net/", accountName)
}
if sasToken != "" {
client, err := azblob.NewClientWithNoCredential(url+"?"+sasToken, nil)
if err != nil {
return nil, fmt.Errorf("init client: %w", err)
}
return &Azure{
client: client,
serviceURL: serviceURL,
sasToken: sasToken,
copyObjectThreshold: copyObjectThreshold,
copySASVersion: copySASVersion,
}, nil
}
if accountName == "" {
// if account name not provided, try to get from env var
accountName = os.Getenv("AZURE_CLIENT_ID")
}
if accountName == "" || accountKey == "" {
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil, fmt.Errorf("init default credentials: %w", err)
}
client, err := azblob.NewClient(url, cred, nil)
if err != nil {
return nil, fmt.Errorf("init client: %w", err)
}
return &Azure{
client: client,
serviceURL: url,
defaultCreds: cred,
copyObjectThreshold: copyObjectThreshold,
copySASVersion: copySASVersion,
}, nil
}
cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
if err != nil {
return nil, fmt.Errorf("init credentials: %w", err)
}
client, err := azblob.NewClientWithSharedKeyCredential(url, cred, nil)
if err != nil {
return nil, fmt.Errorf("init client: %w", err)
}
return &Azure{
client: client,
serviceURL: url,
sharedkeyCreds: cred,
copyObjectThreshold: copyObjectThreshold,
copySASVersion: copySASVersion,
}, nil
}
func (az *Azure) Shutdown() {}
func (az *Azure) String() string {
return "Azure Blob Gateway"
}
func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
meta := map[string]*string{
string(keyAclCapital): backend.GetPtrFromString(encodeBytes(acl)),
string(keyOwnership): backend.GetPtrFromString(encodeBytes([]byte(input.ObjectOwnership))),
}
acct, ok := ctx.Value("bucket-owner").(auth.Account)
if !ok {
acct = auth.Account{}
}
if input.ObjectLockEnabledForBucket != nil && *input.ObjectLockEnabledForBucket {
now := time.Now()
defaultLock := auth.BucketLockConfig{
Enabled: true,
CreatedAt: &now,
}
defaultLockParsed, err := json.Marshal(defaultLock)
if err != nil {
return fmt.Errorf("parse default bucket lock state: %w", err)
}
meta[string(keyBucketLock)] = backend.GetPtrFromString(encodeBytes(defaultLockParsed))
}
tagging, err := backend.ParseCreateBucketTags(input.CreateBucketConfiguration.Tags)
if err != nil {
return err
}
if tagging != nil {
tags, err := json.Marshal(tagging)
if err != nil {
return fmt.Errorf("marshal tags: %w", err)
}
meta[string(keyTags)] = backend.GetPtrFromString(encodeBytes(tags))
}
_, err = az.client.CreateContainer(ctx, *input.Bucket, &container.CreateOptions{Metadata: meta})
if errors.Is(s3err.GetAPIError(s3err.ErrBucketAlreadyExists), azureErrToS3Err(err)) {
aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital))
if err != nil {
return err
}
acl, err := auth.ParseACL(aclBytes)
if err != nil {
return err
}
if acl.Owner == acct.Access {
return s3err.GetBucketErr(s3err.ErrBucketAlreadyOwnedByYou, *input.Bucket)
}
return s3err.GetBucketErr(s3err.ErrBucketAlreadyExists, *input.Bucket)
}
return azureErrToS3Err(err)
}
func (az *Azure) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
opts := &service.ListContainersOptions{
Include: service.ListContainersInclude{
Metadata: true,
},
Prefix: &input.Prefix,
}
pager := az.client.NewListContainersPager(opts)
var buckets []s3response.ListAllMyBucketsEntry
var cToken string
result := s3response.ListAllMyBucketsResult{
Prefix: input.Prefix,
}
outer:
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return result, azureErrToS3Err(err)
}
for _, v := range resp.ContainerItems {
// If we've already filled MaxBuckets, there is a next page — set token and stop
if input.MaxBuckets > 0 && int32(len(buckets)) == input.MaxBuckets {
cToken = buckets[len(buckets)-1].Name
break outer
}
// Skip items at or before the continuation token (client-side "start after")
if input.ContinuationToken != "" && *v.Name <= input.ContinuationToken {
continue
}
if input.IsAdmin {
buckets = append(buckets, s3response.ListAllMyBucketsEntry{
Name: *v.Name,
// TODO: using modification date here instead of creation, is that ok?
CreationDate: *v.Properties.LastModified,
})
} else {
acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
if err != nil {
return result, err
}
if acl.Owner == input.Owner {
buckets = append(buckets, s3response.ListAllMyBucketsEntry{
Name: *v.Name,
// TODO: using modification date here instead of creation, is that ok?
CreationDate: *v.Properties.LastModified,
})
}
}
}
}
result.Buckets.Bucket = buckets
result.Owner.ID = input.Owner
result.ContinuationToken = cToken
return result, nil
}
func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
_, err := az.getContainerMetaData(ctx, *input.Bucket, "any")
if err != nil {
return nil, azureErrToS3Err(err)
}
return &s3.HeadBucketOutput{}, nil
}
func (az *Azure) DeleteBucket(ctx context.Context, bucket string) error {
pager := az.client.NewListBlobsFlatPager(bucket, nil)
for pager.More() {
pg, err := pager.NextPage(ctx)
if err != nil {
return azureErrToS3Err(err)
}
for _, item := range pg.Segment.BlobItems {
// ignore temp multipart objects when determining if bucket non-empty
if !strings.HasPrefix(backend.GetStringFromPtr(item.Name), string(metaTmpMultipartPrefix)) {
return s3err.GetAPIError(s3err.ErrBucketNotEmpty)
}
}
}
_, err := az.client.DeleteContainer(ctx, bucket, nil)
return azureErrToS3Err(err)
}
func (az *Azure) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
return az.setContainerMetaData(ctx, bucket, string(keyOwnership), []byte(ownership))
}
func (az *Azure) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
var ownship types.ObjectOwnership
ownership, err := az.getContainerMetaData(ctx, bucket, string(keyOwnership))
if err != nil {
return ownship, err
}
if len(ownership) == 0 {
return ownship, s3err.GetBucketErr(s3err.ErrOwnershipControlsNotFound, bucket)
}
return types.ObjectOwnership(ownership), nil
}
func (az *Azure) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
return az.deleteContainerMetaData(ctx, bucket, string(keyOwnership))
}
func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
tags, err := backend.ParseObjectTags(getString(po.Tagging))
if err != nil {
return s3response.PutObjectOutput{}, err
}
err = az.evaluateWritePreconditions(ctx, po.Bucket, po.Key, po.IfMatch, po.IfNoneMatch)
if err != nil {
return s3response.PutObjectOutput{}, err
}
metadata := parseMetadata(po.Metadata)
// Store the "Expires" property in the object metadata
if getString(po.Expires) != "" {
if metadata == nil {
metadata = map[string]*string{
string(keyExpires): po.Expires,
}
} else {
metadata[string(keyExpires)] = po.Expires
}
}
if getString(po.WebsiteRedirectLocation) != "" {
if metadata == nil {
metadata = map[string]*string{
string(keyWebsiteRedirect): po.WebsiteRedirectLocation,
}
} else {
metadata[string(keyWebsiteRedirect)] = po.WebsiteRedirectLocation
}
}
opts := &blockblob.UploadStreamOptions{
Metadata: metadata,
Tags: tags,
}
opts.HTTPHeaders = &blob.HTTPHeaders{}
opts.HTTPHeaders.BlobContentEncoding = po.ContentEncoding
opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage
opts.HTTPHeaders.BlobContentDisposition = po.ContentDisposition
opts.HTTPHeaders.BlobContentLanguage = po.ContentLanguage
opts.HTTPHeaders.BlobCacheControl = po.CacheControl
if strings.HasSuffix(*po.Key, "/") {
// Hardcode "application/x-directory" for direcoty objects
opts.HTTPHeaders.BlobContentType = backend.GetPtrFromString(backend.DirContentType)
} else {
opts.HTTPHeaders.BlobContentType = po.ContentType
}
uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, opts)
if err != nil {
return s3response.PutObjectOutput{}, azureErrToS3Err(err)
}
// Set object legal hold
if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
err := az.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) {
err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
}
return s3response.PutObjectOutput{}, err
}
}
// Set object retention
if po.ObjectLockMode != "" {
retention := types.ObjectLockRetention{
Mode: types.ObjectLockRetentionMode(po.ObjectLockMode),
RetainUntilDate: po.ObjectLockRetainUntilDate,
}
retParsed, err := json.Marshal(retention)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("parse object lock retention: %w", err)
}
err = az.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", retParsed)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) {
err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
}
return s3response.PutObjectOutput{}, err
}
}
return s3response.PutObjectOutput{
ETag: convertAzureEtag(uploadResp.ETag),
Size: po.ContentLength,
}, nil
}
func (az *Azure) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error {
if tags == nil {
return az.deleteContainerMetaData(ctx, bucket, string(keyTags))
}
tagsJson, err := json.Marshal(tags)
if err != nil {
return err
}
return az.setContainerMetaData(ctx, bucket, string(keyTags), tagsJson)
}
func (az *Azure) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) {
tagsJson, err := az.getContainerMetaData(ctx, bucket, string(keyTags))
if err != nil {
return nil, err
}
if len(tagsJson) == 0 {
return nil, s3err.GetBucketErr(s3err.ErrBucketTaggingNotFound, bucket)
}
var tags map[string]string
err = json.Unmarshal(tagsJson, &tags)
if err != nil {
return nil, err
}
return tags, nil
}
func (az *Azure) DeleteBucketTagging(ctx context.Context, bucket string) error {
return az.PutBucketTagging(ctx, bucket, nil)
}
func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
client, err := az.getBlobClient(*input.Bucket, *input.Key)
if err != nil {
return nil, err
}
resp, err := client.GetProperties(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
if resp.ETag != nil && resp.LastModified != nil {
err = backend.EvaluatePreconditions(convertAzureEtag(resp.ETag), *resp.LastModified,
backend.PreConditions{
IfMatch: input.IfMatch,
IfNoneMatch: input.IfNoneMatch,
IfModSince: input.IfModifiedSince,
IfUnmodeSince: input.IfUnmodifiedSince,
})
if err != nil {
return nil, err
}
}
var objSize int64
if resp.ContentLength != nil {
objSize = *resp.ContentLength
}
var opts *azblob.DownloadStreamOptions
var partsCount *int32
var contentRange *string
if input.PartNumber != nil {
// Serve a specific part if the object has multipart upload metadata.
// For non-multipart objects (no mp-metadata), partNumber=1 returns the
// full object with no Content-Range; any other partNumber is out of range.
if mpMetaStr, ok := resp.Metadata[string(keyMpMetadata)]; ok && mpMetaStr != nil {
mpMeta, err := backend.UnmarshalMpUploadMetadata([]byte(*mpMetaStr), true)
if err != nil {
return nil, fmt.Errorf("parse object multipart metadata: %w", err)
}
partNum := *input.PartNumber
totalParts := int32(len(mpMeta.Parts))
partsCount = &totalParts
if partNum > totalParts {
return nil, s3err.GetInvalidPartNumberRangeErr(totalParts, partNum)
}
var startOffset int64
if partNum > 1 {
startOffset = mpMeta.Parts[partNum-2]
}
length := mpMeta.Parts[partNum-1] - startOffset
var objSize int64
if resp.ContentLength != nil {
objSize = *resp.ContentLength
}
contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %d-%d/%d", startOffset, startOffset+length-1, objSize))
opts = &azblob.DownloadStreamOptions{
Range: blob.HTTPRange{
Offset: startOffset,
Count: length,
},
}
} else if *input.PartNumber > 1 {
return nil, s3err.GetInvalidPartNumberRangeErr(1, *input.PartNumber)
} else {
// partNumber=1 on a non-multipart object: fall through and serve the
// full object without a range (opts remains nil)
if objSize != 0 {
// if object size is 0, the whole object is served, no content range should be set
contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes 0-%d/%d", objSize-1, objSize))
}
}
} else if *input.Range != "" {
offset, count, isValid, err := backend.ParseObjectRange(objSize, *input.Range)
if err != nil {
return nil, err
}
if isValid {
opts = &azblob.DownloadStreamOptions{
Range: blob.HTTPRange{
Count: count,
Offset: offset,
},
}
contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %v-%v/%v", offset, offset+count-1, objSize))
}
}
blobDownloadResponse, err := az.client.DownloadStream(ctx, *input.Bucket, *input.Key, opts)
if err != nil {
return nil, azureErrToS3Err(err)
}
var tagcount int32
if blobDownloadResponse.TagCount != nil {
tagcount = int32(*blobDownloadResponse.TagCount)
}
return &s3.GetObjectOutput{
AcceptRanges: backend.GetPtrFromString("bytes"),
ContentLength: blobDownloadResponse.ContentLength,
ContentEncoding: blobDownloadResponse.ContentEncoding,
ContentType: blobDownloadResponse.ContentType,
ContentDisposition: blobDownloadResponse.ContentDisposition,
ContentLanguage: blobDownloadResponse.ContentLanguage,
CacheControl: blobDownloadResponse.CacheControl,
ExpiresString: blobDownloadResponse.Metadata[string(keyExpires)],
WebsiteRedirectLocation: blobDownloadResponse.Metadata[string(keyWebsiteRedirect)],
ETag: backend.GetPtrFromString(convertAzureEtag(blobDownloadResponse.ETag)),
LastModified: blobDownloadResponse.LastModified,
Metadata: parseAndFilterAzMetadata(blobDownloadResponse.Metadata),
TagCount: &tagcount,
ContentRange: contentRange,
Body: blobDownloadResponse.Body,
StorageClass: types.StorageClassStandard,
PartsCount: partsCount,
}, nil
}
func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
client, err := az.getBlobClient(*input.Bucket, *input.Key)
if err != nil {
return nil, err
}
resp, err := client.GetProperties(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
if resp.ETag != nil && resp.LastModified != nil {
err = backend.EvaluatePreconditions(convertAzureEtag(resp.ETag), *resp.LastModified,
backend.PreConditions{
IfMatch: input.IfMatch,
IfNoneMatch: input.IfNoneMatch,
IfModSince: input.IfModifiedSince,
IfUnmodeSince: input.IfUnmodifiedSince,
})
if err != nil {
return nil, err
}
}
var size int64
if resp.ContentLength != nil {
size = *resp.ContentLength
}
var contentRange *string
var length int64
var partsCount *int32
if input.PartNumber != nil {
// Serve a specific part if the object has multipart upload metadata.
// For non-multipart objects (no mp-metadata), partNumber=1 returns the
// full object with no Content-Range; any other partNumber is out of range.
if mpMetaStr, ok := resp.Metadata[string(keyMpMetadata)]; ok && mpMetaStr != nil {
mpMeta, err := backend.UnmarshalMpUploadMetadata([]byte(*mpMetaStr), true)
if err != nil {
return nil, fmt.Errorf("parse object multipart metadata: %w", err)
}
partNum := *input.PartNumber
totalParts := int32(len(mpMeta.Parts))
partsCount = &totalParts
if partNum > totalParts {
return nil, s3err.GetInvalidPartNumberRangeErr(totalParts, partNum)
}
var startOffset int64
if partNum > 1 {
startOffset = mpMeta.Parts[partNum-2]
}
length = mpMeta.Parts[partNum-1] - startOffset
contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %d-%d/%d", startOffset, startOffset+length-1, size))
} else if *input.PartNumber > 1 {
return nil, s3err.GetInvalidPartNumberRangeErr(1, *input.PartNumber)
} else {
// partNumber=1 on a non-multipart object: return full object size,
// no Content-Range, no PartsCount.
length = size
if length != 0 {
// if object size is 0, the whole object is served, no content range should be set
contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes 0-%d/%d", size-1, size))
}
}
} else {
startOffset, lgth, isValid, err := backend.ParseObjectRange(size, getString(input.Range))
if err != nil {
return nil, err
}
length = lgth
if isValid {
contentRange = backend.GetPtrFromString(fmt.Sprintf("bytes %v-%v/%v",
startOffset, startOffset+length-1, size))
}
}
result := &s3.HeadObjectOutput{
ContentRange: contentRange,
AcceptRanges: backend.GetPtrFromString("bytes"),
ContentLength: &length,
PartsCount: partsCount,
ContentType: resp.ContentType,
ContentEncoding: resp.ContentEncoding,
ContentLanguage: resp.ContentLanguage,
ContentDisposition: resp.ContentDisposition,
CacheControl: resp.CacheControl,
ExpiresString: resp.Metadata[string(keyExpires)],
WebsiteRedirectLocation: resp.Metadata[string(keyWebsiteRedirect)],
ETag: backend.GetPtrFromString(convertAzureEtag(resp.ETag)),
LastModified: resp.LastModified,
Metadata: parseAndFilterAzMetadata(resp.Metadata),
StorageClass: types.StorageClassStandard,
}
status, ok := resp.Metadata[string(keyObjLegalHold)]
if ok {
if *status == "1" {
result.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
} else {
result.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
}
}
retention, ok := resp.Metadata[string(keyObjRetention)]
if ok {
var config types.ObjectLockRetention
err := json.Unmarshal([]byte(*retention), &config)
if err == nil {
result.ObjectLockMode = types.ObjectLockMode(config.Mode)
result.ObjectLockRetainUntilDate = config.RetainUntilDate
}
}
if resp.TagCount != nil {
tagcount := int32(*resp.TagCount)
result.TagCount = &tagcount
}
return result, nil
}
func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
data, err := az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: input.Bucket,
Key: input.Key,
})
if err != nil {
return s3response.GetObjectAttributesResponse{}, err
}
return s3response.GetObjectAttributesResponse{
ETag: data.ETag,
ObjectSize: data.ContentLength,
StorageClass: data.StorageClass,
LastModified: data.LastModified,
VersionId: data.VersionId,
DeleteMarker: data.DeleteMarker,
}, nil
}
// azMarkerToken is the decoded form of an S3 continuation token. Azure blob
// markers are opaque values that only Azure may mint, so an S3 key can never be
// used as one. The token therefore carries the Azure marker alongside the last
// key of the previous page: the Azure marker resumes the blob listing close to
// where it stopped, and the last key filters out the entries of that Azure page
// which were already returned.
type azMarkerToken struct {
Prefix string `json:"p,omitempty"`
Delimiter string `json:"d,omitempty"`
Marker string `json:"m,omitempty"`
LastKey string `json:"k,omitempty"`
}
// azTokenPrefix marks continuation tokens minted by this backend. Anything
// without it is treated as a plain key, which keeps tokens issued by older
// versions and hand-crafted markers working.
const azTokenPrefix = "vgw1."
func encodeAzMarkerToken(t azMarkerToken) string {
data, err := json.Marshal(t)
if err != nil {
// the struct only holds strings, so this can't fail
return t.LastKey
}
return azTokenPrefix + base64.RawURLEncoding.EncodeToString(data)
}
// decodeAzMarkerToken returns the Azure marker and the last key encoded in the
// continuation token. The Azure marker is only returned for a token minted for
// the same prefix and delimiter, as the marker resumes the underlying blob
// listing and the last key was computed under that request's delimiter; reusing
// either across a different prefix or delimiter would skip the wrong entries.
func decodeAzMarkerToken(token, prefix, delimiter string) (azureMarker, lastKey string) {
if !strings.HasPrefix(token, azTokenPrefix) {
return "", token
}
data, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(token, azTokenPrefix))
if err != nil {
return "", token
}
var t azMarkerToken
if err := json.Unmarshal(data, &t); err != nil {
return "", token
}
if t.Prefix != prefix || t.Delimiter != delimiter {
return "", t.LastKey
}
return t.Marker, t.LastKey
}
// azListingOpts describes a single page of an S3 object listing.
type azListingOpts struct {
prefix string
delimiter string
// marker is the S3 key to start after, exclusive.
marker string
// azureMarker optionally resumes the underlying Azure listing instead of
// walking the prefix from the beginning.
azureMarker string
maxKeys int32
owner string
}
// azListingPage is a page of objects and common prefixes gathered from Azure.
type azListingPage struct {
objects []s3response.Object
commonPrefixes []types.CommonPrefix
isTruncated bool
// lastKey is the last object key or common prefix of the page.
lastKey string
// resumeMarker is the Azure marker of the blob page the listing stopped in.
resumeMarker string
}
// listBlobs collects one S3 listing page from Azure. Azure is always paged with
// its own markers and the delimiter is applied client side, both to match S3
// semantics and because Azure has no equivalent of "start after".
func (az *Azure) listBlobs(ctx context.Context, client *container.Client, opts azListingOpts) (azListingPage, error) {
var page azListingPage
pager := client.NewListBlobsHierarchyPager("", &container.ListBlobsHierarchyOptions{
Prefix: backend.GetPtrFromString(opts.prefix),
Marker: backend.GetPtrFromString(opts.azureMarker),
})
// the Azure marker that started the blob page currently being processed
blobPageMarker := opts.azureMarker
cpSet := make(map[string]struct{})
var pastMax bool
var totalFound int32
loop:
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return azListingPage{}, azureErrToS3Err(err)
}
for _, v := range resp.Segment.BlobItems {
name := backend.GetStringFromPtr(v.Name)
// Filter out multipart upload blobs
if strings.HasPrefix(name, string(metaTmpMultipartPrefix)) {
continue
}
// Apply delimiter logic to determine if this blob contributes to
// a common prefix or is a regular object
key := name
isCP := false
if opts.delimiter != "" {
suffix := strings.TrimPrefix(name, opts.prefix)
before, _, found := strings.Cut(suffix, opts.delimiter)
if found {
isCP = true
key = opts.prefix + before + opts.delimiter
}
}
// Skip everything at or before the marker
if key <= opts.marker {
continue
}
// Deduplicate: multiple blobs can map to the same common prefix
if isCP {
if _, exists := cpSet[key]; exists {
continue
}
}
// A further entry beyond maxKeys means the listing is truncated
if pastMax {
page.isTruncated = true
page.resumeMarker = blobPageMarker
break loop
}
if isCP {
cpSet[key] = struct{}{}
page.commonPrefixes = append(page.commonPrefixes, types.CommonPrefix{
Prefix: backend.GetPtrFromString(key),
})
} else {
page.objects = append(page.objects, s3response.Object{
ETag: backend.GetPtrFromString(convertAzureEtag(v.Properties.ETag)),
Key: v.Name,
LastModified: v.Properties.LastModified,
Size: v.Properties.ContentLength,
StorageClass: types.ObjectStorageClassStandard,
Owner: &types.Owner{
ID: &opts.owner,
},
})
}
page.lastKey = key
totalFound++
if totalFound == opts.maxKeys {
pastMax = true
}
}
blobPageMarker = backend.GetStringFromPtr(resp.NextMarker)
}
return page, nil
}
func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
// Retrieve the bucket acl to get the bucket owner
// All the objects in the bucket are owner by the bucket owner
aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital))
if err != nil {
return s3response.ListObjectsResult{}, azureErrToS3Err(err)
}
acl, err := auth.ParseACL(aclBytes)
if err != nil {
return s3response.ListObjectsResult{}, err
}
client, err := az.getContainerClient(*input.Bucket)
if err != nil {
return s3response.ListObjectsResult{}, nil
}
var maxKeys int32 = defaultListingMaxKeys
if input.MaxKeys != nil {
maxKeys = *input.MaxKeys
}
delimiter := backend.GetStringFromPtr(input.Delimiter)
prefix := backend.GetStringFromPtr(input.Prefix)
effectiveMarker := backend.GetStringFromPtr(input.Marker)
if maxKeys == 0 {
isFalse := false
return s3response.ListObjectsResult{
IsTruncated: &isFalse,
MaxKeys: &maxKeys,
Name: input.Bucket,
Prefix: backend.GetPtrFromString(prefix),
Marker: backend.GetPtrFromString(effectiveMarker),
Delimiter: backend.GetPtrFromString(delimiter),
CommonPrefixes: []types.CommonPrefix{},
}, nil
}
// The S3 marker is an object key, which Azure rejects as a blob listing
// marker, so it is applied client side. ListObjects has no opaque token to
// carry an Azure marker in, hence every page walks the prefix from the
// start; use ListObjectsV2 to page large listings efficiently.
page, err := az.listBlobs(ctx, client, azListingOpts{
prefix: prefix,
delimiter: delimiter,
marker: effectiveMarker,
maxKeys: maxKeys,
owner: acl.Owner,
})
if err != nil {
return s3response.ListObjectsResult{}, err
}
var nextMarker string
if page.isTruncated {
nextMarker = page.lastKey
}
return s3response.ListObjectsResult{
Contents: page.objects,
Marker: backend.GetPtrFromString(effectiveMarker),
MaxKeys: &maxKeys,
Name: input.Bucket,
NextMarker: backend.GetPtrFromString(nextMarker),
Prefix: backend.GetPtrFromString(prefix),
IsTruncated: &page.isTruncated,
Delimiter: backend.GetPtrFromString(delimiter),
CommonPrefixes: page.commonPrefixes,
}, nil
}
func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
// Retrieve the bucket acl to get the bucket owner
// All the objects in the bucket are owner by the bucket owner
aclBytes, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital))
if err != nil {
return s3response.ListObjectsV2Result{}, azureErrToS3Err(err)
}
acl, err := auth.ParseACL(aclBytes)
if err != nil {
return s3response.ListObjectsV2Result{}, err
}
client, err := az.getContainerClient(*input.Bucket)
if err != nil {
return s3response.ListObjectsV2Result{}, nil
}
var maxKeys int32 = defaultListingMaxKeys
if input.MaxKeys != nil {
maxKeys = *input.MaxKeys
}
delimiter := backend.GetStringFromPtr(input.Delimiter)
prefix := backend.GetStringFromPtr(input.Prefix)
startAfterVal := backend.GetStringFromPtr(input.StartAfter)
continuationTokenVal := backend.GetStringFromPtr(input.ContinuationToken)
azureMarker, tokenKey := decodeAzMarkerToken(continuationTokenVal, prefix, delimiter)
// Take the lexicographically larger of startAfter and the continuation
// token key so listing starts strictly after both constraints.
effectiveMarker := startAfterVal
if tokenKey > effectiveMarker {
effectiveMarker = tokenKey
}
if maxKeys == 0 {
isFalse := false
return s3response.ListObjectsV2Result{
IsTruncated: &isFalse,
MaxKeys: &maxKeys,
Name: input.Bucket,
Prefix: backend.GetPtrFromString(prefix),
ContinuationToken: backend.GetPtrFromString(continuationTokenVal),
Delimiter: backend.GetPtrFromString(delimiter),
StartAfter: backend.GetPtrFromString(startAfterVal),
CommonPrefixes: []types.CommonPrefix{},
}, nil
}
// The S3 marker is an object key, which Azure rejects as a blob listing
// marker, so it is applied client side while Azure is paged with the marker
// carried in the continuation token.
page, err := az.listBlobs(ctx, client, azListingOpts{
prefix: prefix,
delimiter: delimiter,
marker: effectiveMarker,
azureMarker: azureMarker,
maxKeys: maxKeys,
owner: acl.Owner,
})
if err != nil {
return s3response.ListObjectsV2Result{}, err
}
var nextToken string
if page.isTruncated {
nextToken = encodeAzMarkerToken(azMarkerToken{
Prefix: prefix,
Delimiter: delimiter,
Marker: page.resumeMarker,
LastKey: page.lastKey,
})
}
keyCount := int32(len(page.objects) + len(page.commonPrefixes))
return s3response.ListObjectsV2Result{
Contents: page.objects,
ContinuationToken: backend.GetPtrFromString(continuationTokenVal),
KeyCount: &keyCount,
MaxKeys: &maxKeys,
Name: input.Bucket,
NextContinuationToken: backend.GetPtrFromString(nextToken),
Prefix: backend.GetPtrFromString(prefix),
IsTruncated: &page.isTruncated,
Delimiter: backend.GetPtrFromString(delimiter),
CommonPrefixes: page.commonPrefixes,
StartAfter: backend.GetPtrFromString(startAfterVal),
}, nil
}
func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
if input.IfMatch != nil || input.IfMatchLastModifiedTime != nil || input.IfMatchSize != nil {
// evaluate the preconditions before deleting the object
props, err := az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: input.Bucket,
Key: input.Key,
})
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
// if object doesn't exist, skip preconditions
// if unexpected error shows up, return the error
return nil, err
}
if err == nil {
var etag string
if props.ETag != nil {
etag = *props.ETag
}
var lastMod time.Time
if props.LastModified != nil {
lastMod = *props.LastModified
}
var size int64
if props.ContentLength != nil {
size = *props.ContentLength
}
err := backend.EvaluateObjectDeletePreconditions(etag, lastMod, size,
backend.ObjectDeletePreconditions{
IfMatch: input.IfMatch,
IfMatchLastModTime: input.IfMatchLastModifiedTime,
IfMatchSize: input.IfMatchSize,
})
if err != nil {
return nil, err
}
}
}
_, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
if err != nil {
azerr, ok := err.(*azcore.ResponseError)
if ok && azerr.StatusCode == 404 {
// if the object does not exist, S3 returns success
return &s3.DeleteObjectOutput{}, nil
}
}
return &s3.DeleteObjectOutput{}, azureErrToS3Err(err)
}
func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
delResult, errs := []types.DeletedObject{}, []types.Error{}
for _, obj := range input.Delete.Objects {
_, err := az.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: input.Bucket,
Key: obj.Key,
})
if err == nil {
delResult = append(delResult, types.DeletedObject{Key: obj.Key})
} else {
errs = append(errs, s3err.ObjectDeleteError(obj.Key, obj.VersionId, err))
}
}
return s3response.DeleteResult{
Deleted: delResult,
Error: errs,
}, nil
}
func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
dstClient, err := az.getBlobClient(*input.Bucket, *input.Key)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
if input.ExpectedSourceBucketOwner != nil && *input.ExpectedSourceBucketOwner != "" {
aclData, err := az.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
if err != nil {
return s3response.CopyObjectOutput{}, err
}
srcAcl, err := auth.ParseACL(aclData)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
if srcAcl.Owner != *input.ExpectedSourceBucketOwner {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
if !areNils(input.CopySourceIfMatch, input.CopySourceIfNoneMatch) || !areNils(input.CopySourceIfModifiedSince, input.CopySourceIfUnmodifiedSince) {
_, err = az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &srcBucket,
Key: &srcObj,
IfMatch: input.CopySourceIfMatch,
IfNoneMatch: input.CopySourceIfNoneMatch,
IfModifiedSince: input.CopySourceIfModifiedSince,
IfUnmodifiedSince: input.CopySourceIfUnmodifiedSince,
})
if err != nil {
return s3response.CopyObjectOutput{}, err
}
}
if srcBucket == *input.Bucket && srcObj == *input.Key {
if input.MetadataDirective != types.MetadataDirectiveReplace {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
}
resp, err := dstClient.GetProperties(ctx, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
if resp.ContentLength != nil && *resp.ContentLength > az.copyObjectThreshold {
return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold)
}
// Set object meta http headers
res, err := dstClient.SetHTTPHeaders(ctx, blob.HTTPHeaders{
BlobCacheControl: input.CacheControl,
BlobContentDisposition: input.ContentDisposition,
BlobContentEncoding: input.ContentEncoding,
BlobContentLanguage: input.ContentLanguage,
BlobContentType: input.ContentType,
}, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
meta := input.Metadata
if meta == nil {
meta = make(map[string]string)
}
// Embed "Expires" in object metadata
if getString(input.Expires) != "" {
meta[string(keyExpires)] = *input.Expires
}
if getString(input.WebsiteRedirectLocation) != "" {
meta[string(keyWebsiteRedirect)] = *input.WebsiteRedirectLocation
}
// Set object metadata
_, err = dstClient.SetMetadata(ctx, parseMetadata(meta), nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
// Set object legal hold
if input.ObjectLockLegalHoldStatus != "" {
err = az.PutObjectLegalHold(ctx, *input.Bucket, *input.Key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) {
err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
}
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
}
// Set object retention
if input.ObjectLockMode != "" && input.ObjectLockRetainUntilDate != nil {
retention := s3response.PutObjectRetentionInput{
Mode: types.ObjectLockRetentionMode(input.ObjectLockMode),
RetainUntilDate: s3response.AmzDate{
Time: *input.ObjectLockRetainUntilDate,
},
}
retParsed, err := json.Marshal(retention)
if err != nil {
return s3response.CopyObjectOutput{}, fmt.Errorf("parse object retention: %w", err)
}
err = az.PutObjectRetention(ctx, *input.Bucket, *input.Key, "", retParsed)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) {
err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
}
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
}
// Set object Tagging, if tagging directive is "REPLACE"
if input.TaggingDirective == types.TaggingDirectiveReplace {
tags, err := backend.ParseObjectTags(getString(input.Tagging))
if err != nil {
return s3response.CopyObjectOutput{}, err
}
_, err = dstClient.SetTags(ctx, tags, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
}
return s3response.CopyObjectOutput{
CopyObjectResult: &s3response.CopyObjectResult{
LastModified: res.LastModified,
ETag: backend.GetPtrFromString(convertAzureEtag(res.ETag)),
},
}, nil
}
// Cross-object copy: try server-side copy first, fall back to download+reupload.
srcClient, err := az.getBlobClient(srcBucket, srcObj)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
srcProps, err := srcClient.GetProperties(ctx, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
if srcProps.ContentLength != nil && *srcProps.ContentLength > az.copyObjectThreshold {
return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold)
}
out, err := az.serverSideCopyObject(ctx, input, srcBucket, srcObj, dstClient, srcClient, &srcProps)
if err == nil {
return out, nil
}
if !errors.Is(err, errServerSideCopyFallback) {
return s3response.CopyObjectOutput{}, err
}
debuglogger.Logf("falling back to download+reupload (%q/%q -> %q/%q): %v",
srcBucket, srcObj, *input.Bucket, *input.Key, err)
// Fallback: download and re-upload through the gateway.
downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
defer downloadResp.Body.Close()
if downloadResp.ContentLength != nil && *downloadResp.ContentLength > az.copyObjectThreshold {
return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold)
}
pInput := s3response.PutObjectInput{
Body: downloadResp.Body,
Bucket: input.Bucket,
Key: input.Key,
ContentLength: downloadResp.ContentLength,
ContentType: input.ContentType,
ContentEncoding: input.ContentEncoding,
ContentDisposition: input.ContentDisposition,
ContentLanguage: input.ContentLanguage,
CacheControl: input.CacheControl,
Expires: input.Expires,
WebsiteRedirectLocation: input.WebsiteRedirectLocation,
Metadata: input.Metadata,
ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate,
ObjectLockMode: input.ObjectLockMode,
ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus,
}
if input.MetadataDirective == types.MetadataDirectiveCopy {
// Expires is in downloadResp.Metadata
pInput.Expires = nil
pInput.CacheControl = downloadResp.CacheControl
pInput.ContentDisposition = downloadResp.ContentDisposition
pInput.ContentEncoding = downloadResp.ContentEncoding
pInput.ContentLanguage = downloadResp.ContentLanguage
pInput.ContentType = downloadResp.ContentType
pInput.Metadata = parseAzMetadata(downloadResp.Metadata)
delete(pInput.Metadata, string(keyWebsiteRedirect))
}
if input.TaggingDirective == types.TaggingDirectiveReplace {
pInput.Tagging = input.Tagging
}
// Create the destination object
resp, err := az.PutObject(ctx, pInput)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
// Copy the object tagging, if tagging directive is "COPY"
if input.TaggingDirective == types.TaggingDirectiveCopy {
srcClient, err := az.getBlobClient(srcBucket, srcObj)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
res, err := srcClient.GetTags(ctx, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
_, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
}
return s3response.CopyObjectOutput{
CopyObjectResult: &s3response.CopyObjectResult{
ETag: &resp.ETag,
},
}, nil
}
func (az *Azure) PutObjectTagging(ctx context.Context, bucket, object, _ string, tags map[string]string) error {
client, err := az.getBlobClient(bucket, object)
if err != nil {
return err
}
_, err = client.SetTags(ctx, tags, nil)
if err != nil {
return azureErrToS3Err(err)
}
return nil
}
func (az *Azure) GetObjectTagging(ctx context.Context, bucket, object, _ string) (map[string]string, error) {
client, err := az.getBlobClient(bucket, object)
if err != nil {
return nil, err
}
tags, err := client.GetTags(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
return parseAzTags(tags.BlobTagSet), nil
}
func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object, _ string) error {
client, err := az.getBlobClient(bucket, object)
if err != nil {
return err
}
_, err = client.SetTags(ctx, map[string]string{}, nil)
if err != nil {
return azureErrToS3Err(err)
}
return nil
}
func (az *Azure) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
if input.ObjectLockLegalHoldStatus != "" || input.ObjectLockMode != "" {
bucketLock, err := az.getContainerMetaData(ctx, *input.Bucket, string(keyBucketLock))
if err != nil {
return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err)
}
if len(bucketLock) == 0 {
return s3response.InitiateMultipartUploadResult{},
s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
}
var bucketLockConfig auth.BucketLockConfig
err = json.Unmarshal(bucketLock, &bucketLockConfig)
if err != nil {
return s3response.InitiateMultipartUploadResult{},
fmt.Errorf("parse bucket lock config: %w", err)
}
if !bucketLockConfig.Enabled {
return s3response.InitiateMultipartUploadResult{},
s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
}
}
meta := parseMetadata(input.Metadata)
meta[string(onameAttr)] = input.Key
if getString(input.Expires) != "" {
meta[string(keyExpires)] = input.Expires
}
if getString(input.WebsiteRedirectLocation) != "" {
meta[string(keyWebsiteRedirect)] = input.WebsiteRedirectLocation
}
// parse object tags
tags, err := backend.ParseObjectTags(getString(input.Tagging))
if err != nil {
return s3response.InitiateMultipartUploadResult{}, err
}
// set blob legal hold status in metadata
if input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn {
meta[string(keyObjLegalHold)] = backend.GetPtrFromString("1")
}
// set blob retention date
if input.ObjectLockMode != "" {
retention := types.ObjectLockRetention{
Mode: types.ObjectLockRetentionMode(input.ObjectLockMode),
RetainUntilDate: input.ObjectLockRetainUntilDate,
}
retParsed, err := json.Marshal(retention)
if err != nil {
return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err)
}
meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retParsed))
}
uploadId := uuid.New().String()
tmpPath := createMetaTmpPath(*input.Key, uploadId)
opts := &blockblob.UploadBufferOptions{
Metadata: meta,
Tags: tags,
HTTPHeaders: &blob.HTTPHeaders{
BlobContentType: input.ContentType,
BlobContentEncoding: input.ContentEncoding,
BlobCacheControl: input.CacheControl,
BlobContentDisposition: input.ContentDisposition,
BlobContentLanguage: input.ContentLanguage,
},
}
// Create and empty blob in .sgwtmp/multipart/<uploadId>/<object hash>
// The blob indicates multipart upload initialization and holds the mp metadata
// e.g tagging, content-type, metadata, object lock status ...
_, err = az.client.UploadBuffer(ctx, *input.Bucket, tmpPath, []byte{}, opts)
if err != nil {
return s3response.InitiateMultipartUploadResult{}, azureErrToS3Err(err)
}
return s3response.InitiateMultipartUploadResult{
Bucket: *input.Bucket,
Key: *input.Key,
UploadId: uploadId,
}, nil
}
// Each part is translated into an uncommitted block in a newly created blob in staging area
func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId)
if err != nil {
return nil, err
}
// TODO: request streamable version of StageBlock()
// (*blockblob.Client).StageBlock does not have a streamable
// version of this function at this time, so we need to cache
// the body in memory to create an io.ReadSeekCloser
rdr, err := getReadSeekCloser(input.Body)
if err != nil {
return nil, err
}
// block id serves as etag here
etag := blockIDInt32ToBase64(*input.PartNumber)
quotedETag := quoteETag(etag)
// Azure StageBlock rejects Content-Length: 0 as an invalid header value.
// Track zero-byte parts in the sgwtmp metadata instead of staging them.
size, err := rdr.Seek(0, io.SeekEnd)
if err != nil {
return nil, err
}
if _, err = rdr.Seek(0, io.SeekStart); err != nil {
return nil, err
}
if size == 0 {
err := az.trackZeroBytePart(ctx, *input.Bucket, *input.Key, *input.UploadId, *input.PartNumber)
if err != nil {
return nil, err
}
return &s3.UploadPartOutput{ETag: &quotedETag}, nil
}
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return nil, err
}
_, err = client.StageBlock(ctx, etag, rdr, nil)
if err != nil {
return nil, parseMpError(err)
}
return &s3.UploadPartOutput{
ETag: &quotedETag,
}, nil
}
func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return s3response.CopyPartResult{}, err
}
err = az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId)
if err != nil {
return s3response.CopyPartResult{}, err
}
eTag := blockIDInt32ToBase64(*input.PartNumber)
//TODO: handle block copy by range
//TODO: the action returns not implemented on azurite, maybe in production this will work?
_, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil)
if err != nil {
return s3response.CopyPartResult{}, parseMpError(err)
}
// The staged block id serves as the part ETag, returned quoted to match
// the S3 contract and the form UploadPart/ListParts emit.
quotedETag := quoteETag(eTag)
return s3response.CopyPartResult{
ETag: &quotedETag,
LastModified: time.Now(),
}, nil
}
// Lists all uncommitted parts from the blob
func (az *Azure) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId)
if err != nil {
return s3response.ListPartsResult{}, err
}
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return s3response.ListPartsResult{}, nil
}
var partNumberMarker int
var nextPartNumberMarker int
var maxParts int32 = math.MaxInt32
var isTruncated bool
if *input.PartNumberMarker != "" {
partNumberMarker, err = strconv.Atoi(*input.PartNumberMarker)
if err != nil {
return s3response.ListPartsResult{},
s3err.GetInvalidArgMaxLimiter("part-number-marker", *input.PartNumberMarker)
}
}
if input.MaxParts != nil {
maxParts = *input.MaxParts
}
resp, blockListErr := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
if blockListErr != nil {
if !errors.Is(azureErrToS3Err(blockListErr), s3err.GetAPIError(s3err.ErrNoSuchKey)) {
return s3response.ListPartsResult{}, blockListErr
}
// NoSuchKey means no blocks have been staged yet (possible if only zero-byte
// parts exist). Continue so we can still return those from metadata.
}
parts := []s3response.Part{}
if blockListErr == nil {
for _, el := range resp.UncommittedBlocks {
partNumber, err := decodeBlockId(*el.Name)
if err != nil {
return s3response.ListPartsResult{}, err
}
if partNumberMarker >= partNumber {
continue
}
parts = append(parts, s3response.Part{
Size: *el.Size,
ETag: quoteETag(*el.Name),
PartNumber: partNumber,
LastModified: time.Now(),
})
}
}
// Merge in zero-byte parts tracked in the sgwtmp metadata.
zbParts, _ := az.getZeroByteParts(ctx, *input.Bucket, *input.Key, *input.UploadId)
for _, zbPartNum := range zbParts {
if partNumberMarker >= int(zbPartNum) {
continue
}
parts = append(parts, s3response.Part{
Size: 0,
ETag: quoteETag(blockIDInt32ToBase64(zbPartNum)),
PartNumber: int(zbPartNum),
LastModified: time.Now(),
})
}
// Sort by part number and apply maxParts limit.
sort.Slice(parts, func(i, j int) bool {
return parts[i].PartNumber < parts[j].PartNumber
})
if int32(len(parts)) > maxParts {
parts = parts[:maxParts]
nextPartNumberMarker = parts[len(parts)-1].PartNumber
isTruncated = true
}
return s3response.ListPartsResult{
Bucket: *input.Bucket,
Key: *input.Key,
Parts: parts,
NextPartNumberMarker: nextPartNumberMarker,
PartNumberMarker: partNumberMarker,
IsTruncated: isTruncated,
MaxParts: int(maxParts),
StorageClass: types.StorageClassStandard,
}, nil
}
// Lists all the multipart uploads initiated with .sgwtmp/multipart prefix
func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
var bucket string
if input.Bucket != nil {
bucket = *input.Bucket
}
client, err := az.getContainerClient(bucket)
if err != nil {
return s3response.ListMultipartUploadsResult{}, err
}
var delimiter string
if input.Delimiter != nil {
delimiter = *input.Delimiter
}
var prefix string
if input.Prefix != nil {
prefix = *input.Prefix
}
var keyMarker string
if input.KeyMarker != nil {
keyMarker = *input.KeyMarker
}
var uploadIDMarker string
if input.UploadIdMarker != nil {
uploadIDMarker = *input.UploadIdMarker
}
maxUploads := int(*input.MaxUploads)
mpPrefix := string(metaTmpMultipartPrefix)
pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Include: container.ListBlobsInclude{Metadata: true},
Prefix: &mpPrefix,
})
uploads := []s3response.Upload{}
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return s3response.ListMultipartUploadsResult{}, azureErrToS3Err(err)
}
for _, el := range resp.Segment.BlobItems {
key, ok := el.Metadata[string(onameAttrLower)]
if !ok {
continue
}
if keyMarker != "" && *key <= keyMarker {
continue
}
if prefix != "" && !strings.HasPrefix(*key, prefix) {
continue
}
path := filepath.Clean(*el.Name)
parts := strings.Split(path, "/")
uploadId := parts[2]
uploads = append(uploads, s3response.Upload{
Key: *key,
Initiated: *el.Properties.CreationTime,
UploadID: uploadId,
StorageClass: types.StorageClassStandard,
})
}
}
// Sort once: Key asc, Initiated asc
sort.SliceStable(uploads, func(i, j int) bool {
if uploads[i].Key != uploads[j].Key {
return uploads[i].Key < uploads[j].Key
}
return uploads[i].Initiated.Before(uploads[j].Initiated)
})
result, err := backend.ListMultipartUploads(uploads, prefix, delimiter, keyMarker, uploadIDMarker, maxUploads)
if err != nil {
return s3response.ListMultipartUploadsResult{}, err
}
return s3response.ListMultipartUploadsResult{
Bucket: bucket,
Delimiter: delimiter,
KeyMarker: keyMarker,
MaxUploads: maxUploads,
Prefix: prefix,
NextKeyMarker: result.NextKeyMarker,
NextUploadIDMarker: result.NextUploadIDMarker,
UploadIDMarker: uploadIDMarker,
IsTruncated: result.IsTruncated,
Uploads: result.Uploads,
CommonPrefixes: result.CommonPrefixes,
}, nil
}
// Deletes the block blob with committed/uncommitted blocks
// Cleans up the initiated multipart upload in .sgwtmp namespace
func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
if input.IfMatchInitiatedTime != nil {
client, err := az.getBlobClient(*input.Bucket, tmpPath)
if err != nil {
return err
}
resp, err := client.GetProperties(ctx, nil)
if err != nil {
return azureErrToS3Err(err)
}
if resp.LastModified != nil && resp.LastModified.Unix() != input.IfMatchInitiatedTime.Unix() {
return s3err.GetPreconditionFailedErr(s3err.ConditionIfMatchInitiatedTime)
}
}
_, err := az.client.DeleteBlob(ctx, *input.Bucket, tmpPath, nil)
if err != nil {
return parseMpError(err)
}
// Cleanup the uploaded parts
_, err = az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
if err != nil {
err = azureErrToS3Err(err)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
return nil
}
return err
}
return nil
}
// Commits all the uncommitted blocks inside the block blob
// And moves the block blob from staging area into the blobs list.
// Copeies the multipart metadata from .sgwtmp namespace into the newly created blob
// Deletes the multipart upload 'blob' from .sgwtmp namespace
// It indicates the end of the multipart upload
func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
var res s3response.CompleteMultipartUploadResult
err := az.evaluateWritePreconditions(ctx, input.Bucket, input.Key, input.IfMatch, input.IfNoneMatch)
if err != nil {
return s3response.CompleteMultipartUploadResult{}, "", err
}
tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
blobClient, err := az.getBlobClient(*input.Bucket, tmpPath)
if err != nil {
return res, "", err
}
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
mpErr := parseMpError(err)
// If the tmp blob is already gone, the upload may have already been
// completed. Check the final object's mp-metadata and return success
// if the upload IDs match.
if errors.Is(mpErr, s3err.GetAPIError(s3err.ErrNoSuchUpload)) {
finalClient, clientErr := az.getBlobClient(*input.Bucket, *input.Key)
if clientErr == nil {
finalProps, propErr := finalClient.GetProperties(ctx, nil)
if propErr == nil {
if mpMetaStr, ok := finalProps.Metadata[string(keyMpMetadata)]; ok && mpMetaStr != nil {
mpMeta, metaErr := backend.UnmarshalMpUploadMetadata([]byte(*mpMetaStr), true)
if metaErr == nil && mpMeta.UploadID == *input.UploadId {
return s3response.CompleteMultipartUploadResult{
Bucket: input.Bucket,
Key: input.Key,
ETag: backend.GetPtrFromString(convertAzureEtag(finalProps.ETag)),
}, "", nil
}
}
}
}
}
return res, "", mpErr
}
tags, err := blobClient.GetTags(ctx, nil)
if err != nil {
return res, "", parseMpError(err)
}
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return res, "", err
}
blockIds := []string{}
blockList, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil)
if err != nil {
if !errors.Is(azureErrToS3Err(err), s3err.GetAPIError(s3err.ErrNoSuchKey)) {
return res, "", azureErrToS3Err(err)
}
// NoSuchKey: no blocks staged; only zero-byte parts may exist.
}
// Collect zero-byte parts tracked in the sgwtmp metadata.
zbParts, err := az.getZeroByteParts(ctx, *input.Bucket, *input.Key, *input.UploadId)
if err != nil {
return res, "", err
}
zbPartsMap := make(map[int32]bool, len(zbParts))
for _, p := range zbParts {
zbPartsMap[p] = true
}
if len(blockList.UncommittedBlocks)+len(zbParts) != len(input.MultipartUpload.Parts) {
return res, "", s3err.GetInvalidPartErr(*input.UploadId, 0, "")
}
uncommittedBlocks := map[int32]*blockblob.Block{}
for _, el := range blockList.UncommittedBlocks {
ptNumber, err := decodeBlockId(backend.GetStringFromPtr(el.Name))
if err != nil {
return res, "", fmt.Errorf("invalid block name: %w", err)
}
uncommittedBlocks[int32(ptNumber)] = el
}
// The initial value is the lower limit of partNumber: 0
var totalSize int64
var partNumber int32
// partSizes[i] = cumulative byte offset after part i+1 (see backend.MpUploadMetadata)
var partSizes []int64
last := len(input.MultipartUpload.Parts) - 1
for i, part := range input.MultipartUpload.Parts {
if part.PartNumber == nil {
return res, "", s3err.GetAPIError(s3err.ErrMalformedXML)
}
if part.ETag == nil {
return res, "", s3err.GetAPIError(s3err.ErrMalformedXML)
}
// Clients may submit the part ETag in quoted form (as returned by
// UploadPart/ListParts) or raw; normalize before comparing to the
// raw Azure block id.
clientETag := getString(backend.TrimEtag(part.ETag))
if *part.PartNumber < 1 {
return res, "", s3err.GetInvalidArgumentErr(s3err.InvalidArgCompleteMpPartNumber, fmt.Sprint(*part.PartNumber))
}
if *part.PartNumber <= partNumber {
return res, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder)
}
partNumber = *part.PartNumber
block, ok := uncommittedBlocks[*part.PartNumber]
if !ok {
// Check if this is a tracked zero-byte part.
if zbPartsMap[*part.PartNumber] {
expectedETag := blockIDInt32ToBase64(*part.PartNumber)
if clientETag != expectedETag {
return res, "", s3err.GetInvalidPartErr(*input.UploadId, *part.PartNumber, expectedETag)
}
// Non-last zero-byte parts violate the minimum part size.
if i < last {
return res, "", s3err.GetEntityTooSmallErr(0, backend.MinPartSize)
}
// Zero-byte parts contribute no data; skip adding to blockIds.
partSizes = append(partSizes, totalSize)
continue
}
return res, "", s3err.GetInvalidPartErr(*input.UploadId, *part.PartNumber, "")
}
if clientETag != *block.Name {
return res, "", s3err.GetInvalidPartErr(*input.UploadId, *part.PartNumber, clientETag)
}
// all parts except the last need to be greater, than
// the minimum allowed size (5 Mib)
if i < last && *block.Size < backend.MinPartSize {
return res, "", s3err.GetEntityTooSmallErr(*block.Size, backend.MinPartSize)
}
totalSize += *block.Size
partSizes = append(partSizes, totalSize)
blockIds = append(blockIds, *block.Name)
}
if input.MpuObjectSize != nil && totalSize != *input.MpuObjectSize {
return res, "",
s3err.GetIncorrectMpObjectSizeErr(totalSize, *input.MpuObjectSize)
}
// Remove internal tracking keys from metadata before storing on the final blob.
delete(props.Metadata, string(keyMpZeroBytesParts))
// Serialize multipart metadata so GetObject/HeadObject can serve by part-number.
mpMeta := backend.MpUploadMetadata{UploadID: *input.UploadId, Parts: partSizes}
mpMetaBytes, err := backend.MarshalMpUploadMetadata(mpMeta, true)
if err != nil {
return res, "", fmt.Errorf("marshal mp metadata: %w", err)
}
mpMetaStr := string(mpMetaBytes)
if props.Metadata == nil {
props.Metadata = map[string]*string{}
}
props.Metadata[string(keyMpMetadata)] = &mpMetaStr
opts := &blockblob.CommitBlockListOptions{
Metadata: props.Metadata,
Tags: parseAzTags(tags.BlobTagSet),
}
opts.HTTPHeaders = &blob.HTTPHeaders{
BlobContentType: props.ContentType,
BlobContentEncoding: props.ContentEncoding,
BlobContentDisposition: props.ContentDisposition,
BlobContentLanguage: props.ContentLanguage,
BlobCacheControl: props.CacheControl,
}
resp, err := client.CommitBlockList(ctx, blockIds, opts)
if err != nil {
return res, "", parseMpError(err)
}
// cleanup the multipart upload
_, err = blobClient.Delete(ctx, nil)
if err != nil {
return res, "", parseMpError(err)
}
return s3response.CompleteMultipartUploadResult{
Bucket: input.Bucket,
Key: input.Key,
ETag: backend.GetPtrFromString(convertAzureEtag(resp.ETag)),
}, "", nil
}
func (az *Azure) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
return az.setContainerMetaData(ctx, bucket, string(keyAclCapital), data)
}
func (az *Azure) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
return az.getContainerMetaData(ctx, *input.Bucket, string(keyAclCapital))
}
func (az *Azure) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
if policy == nil {
return az.deleteContainerMetaData(ctx, bucket, string(keyPolicy))
}
return az.setContainerMetaData(ctx, bucket, string(keyPolicy), policy)
}
func (az *Azure) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
p, err := az.getContainerMetaData(ctx, bucket, string(keyPolicy))
if err != nil {
return nil, err
}
if len(p) == 0 {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucketPolicy, bucket)
}
return p, nil
}
func (az *Azure) DeleteBucketPolicy(ctx context.Context, bucket string) error {
return az.PutBucketPolicy(ctx, bucket, nil)
}
func (az *Azure) PutBucketCors(ctx context.Context, bucket string, cors []byte) error {
if cors == nil {
return az.deleteContainerMetaData(ctx, bucket, string(keyCors))
}
return az.setContainerMetaData(ctx, bucket, string(keyCors), cors)
}
func (az *Azure) GetBucketCors(ctx context.Context, bucket string) ([]byte, error) {
p, err := az.getContainerMetaData(ctx, bucket, string(keyCors))
if err != nil {
return nil, err
}
if len(p) == 0 {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchCORSConfiguration, bucket)
}
return p, nil
}
func (az *Azure) DeleteBucketCors(ctx context.Context, bucket string) error {
return az.PutBucketCors(ctx, bucket, nil)
}
func (az *Azure) PutBucketWebsite(ctx context.Context, bucket string, website []byte) error {
if website == nil {
return az.deleteContainerMetaData(ctx, bucket, string(keyWebsite))
}
encoded, err := backend.MarshalWebsiteConfig(website, true)
if err != nil {
return err
}
return az.setContainerMetaData(ctx, bucket, string(keyWebsite), encoded)
}
func (az *Azure) GetBucketWebsite(ctx context.Context, bucket string) ([]byte, error) {
website, err := az.getContainerMetaData(ctx, bucket, string(keyWebsite))
if err != nil {
return nil, err
}
if len(website) == 0 {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchWebsiteConfiguration, bucket)
}
decoded, err := backend.UnmarshalWebsiteConfig(website, true)
if err != nil {
return nil, err
}
return decoded, nil
}
func (az *Azure) DeleteBucketWebsite(ctx context.Context, bucket string) error {
return az.PutBucketWebsite(ctx, bucket, nil)
}
func (az *Azure) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
return az.setContainerMetaData(ctx, bucket, string(keyBucketLock), config)
}
func (az *Azure) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
if err != nil {
return nil, err
}
if len(cfg) == 0 {
return nil, s3err.GetBucketErr(s3err.ErrObjectLockConfigurationNotFound, bucket)
}
return cfg, nil
}
func (az *Azure) PutObjectRetention(ctx context.Context, bucket, object, versionId string, retention []byte) error {
err := az.isBucketObjectLockEnabled(ctx, bucket)
if err != nil {
return err
}
blobClient, err := az.getBlobClient(bucket, object)
if err != nil {
return err
}
blobProps, err := blobClient.GetProperties(ctx, nil)
if err != nil {
return azureErrToS3Err(err)
}
meta := blobProps.Metadata
if meta == nil {
meta = map[string]*string{
string(keyObjRetention): backend.GetPtrFromString(string(retention)),
}
} else {
meta[string(keyObjRetention)] = backend.GetPtrFromString(string(retention))
}
_, err = blobClient.SetMetadata(ctx, meta, nil)
if err != nil {
return azureErrToS3Err(err)
}
return nil
}
func (az *Azure) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
client, err := az.getBlobClient(bucket, object)
if err != nil {
return nil, err
}
props, err := client.GetProperties(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
err = az.isBucketObjectLockEnabled(ctx, bucket)
if err != nil {
return nil, err
}
retentionPtr, ok := props.Metadata[string(keyObjRetention)]
if !ok {
return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
}
return []byte(*retentionPtr), nil
}
func (az *Azure) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
err := az.isBucketObjectLockEnabled(ctx, bucket)
if err != nil {
return err
}
blobClient, err := az.getBlobClient(bucket, object)
if err != nil {
return err
}
blobProps, err := blobClient.GetProperties(ctx, nil)
if err != nil {
return azureErrToS3Err(err)
}
var statusData string
if status {
statusData = "1"
} else {
statusData = "0"
}
meta := blobProps.Metadata
if meta == nil {
meta = map[string]*string{
string(keyObjLegalHold): &statusData,
}
} else {
meta[string(keyObjLegalHold)] = &statusData
}
_, err = blobClient.SetMetadata(ctx, meta, nil)
if err != nil {
return azureErrToS3Err(err)
}
return nil
}
func (az *Azure) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
client, err := az.getBlobClient(bucket, object)
if err != nil {
return nil, err
}
props, err := client.GetProperties(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
err = az.isBucketObjectLockEnabled(ctx, bucket)
if err != nil {
return nil, err
}
retentionPtr, ok := props.Metadata[string(keyObjLegalHold)]
if !ok {
return nil, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)
}
status := *retentionPtr == "1"
return &status, nil
}
func (az *Azure) ChangeBucketOwner(ctx context.Context, bucket, owner string) error {
return auth.UpdateBucketACLOwner(ctx, az, bucket, owner)
}
// The action actually returns the containers owned by the user, who initialized the gateway
// TODO: Not sure if there's a way to list all the containers and owners?
func (az *Azure) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) {
opts := &service.ListContainersOptions{
Include: service.ListContainersInclude{Metadata: true},
}
pager := az.client.NewListContainersPager(opts)
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return buckets, azureErrToS3Err(err)
}
for _, v := range resp.ContainerItems {
acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
if err != nil {
return buckets, err
}
buckets = append(buckets, s3response.Bucket{
Name: *v.Name,
Owner: acl.Owner,
})
}
}
return buckets, nil
}
func (az *Azure) isBucketObjectLockEnabled(ctx context.Context, bucket string) error {
cfg, err := az.getContainerMetaData(ctx, bucket, string(keyBucketLock))
if err != nil {
return azureErrToS3Err(err)
}
if len(cfg) == 0 {
return s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)
}
var bucketLockConfig auth.BucketLockConfig
err = json.Unmarshal(cfg, &bucketLockConfig)
if err != nil {
return fmt.Errorf("parse bucket lock config: %w", err)
}
if !bucketLockConfig.Enabled {
return s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)
}
return nil
}
func (az *Azure) getContainerURL(cntr string) string {
return fmt.Sprintf("%v/%v", strings.TrimRight(az.serviceURL, "/"), cntr)
}
func (az *Azure) getBlobURL(cntr, blb string) string {
// URL-encode the blob name to handle special characters like {, }, #, spaces, etc.
// Use PathEscape to encode the blob name while preserving forward slashes
encodedBlob := url.PathEscape(blb)
return fmt.Sprintf("%v/%v", az.getContainerURL(cntr), encodedBlob)
}
func (az *Azure) getBlobClient(cntr, blb string) (*blob.Client, error) {
blobURL := az.getBlobURL(cntr, blb)
if az.defaultCreds != nil {
return blob.NewClient(blobURL, az.defaultCreds, nil)
}
if az.sasToken != "" {
return blob.NewClientWithNoCredential(blobURL+"?"+az.sasToken, nil)
}
return blob.NewClientWithSharedKeyCredential(blobURL, az.sharedkeyCreds, nil)
}
func (az *Azure) getContainerClient(cntr string) (*container.Client, error) {
containerURL := az.getContainerURL(cntr)
if az.defaultCreds != nil {
return container.NewClient(containerURL, az.defaultCreds, nil)
}
if az.sasToken != "" {
return container.NewClientWithNoCredential(containerURL+"?"+az.sasToken, nil)
}
return container.NewClientWithSharedKeyCredential(containerURL, az.sharedkeyCreds, nil)
}
func (az *Azure) getBlockBlobClient(cntr, blb string) (*blockblob.Client, error) {
blobURL := az.getBlobURL(cntr, blb)
if az.defaultCreds != nil {
return blockblob.NewClient(blobURL, az.defaultCreds, nil)
}
if az.sasToken != "" {
return blockblob.NewClientWithNoCredential(blobURL+"?"+az.sasToken, nil)
}
return blockblob.NewClientWithSharedKeyCredential(blobURL, az.sharedkeyCreds, nil)
}
func parseMetadata(m map[string]string) map[string]*string {
if m == nil {
return nil
}
meta := make(map[string]*string)
for k, v := range m {
val := v
meta[k] = &val
}
return meta
}
func parseAndFilterAzMetadata(m map[string]*string) map[string]string {
if m == nil {
return nil
}
keywords := keyTags.Table()
meta := make(map[string]string)
for k, v := range m {
_, ok := keywords[strings.ToLower(k)]
if ok {
continue
}
meta[k] = *v
}
return meta
}
func parseAzMetadata(m map[string]*string) map[string]string {
if m == nil {
return nil
}
meta := make(map[string]string)
for k, v := range m {
meta[k] = *v
}
return meta
}
func parseAzTags(tagSet []*blob.Tags) map[string]string {
tags := map[string]string{}
for _, tag := range tagSet {
tags[*tag.Key] = *tag.Value
}
return tags
}
func getString(str *string) string {
if str == nil {
return ""
}
return *str
}
// Converts io.Reader into io.ReadSeekCloser
func getReadSeekCloser(input io.Reader) (io.ReadSeekCloser, error) {
var buffer bytes.Buffer
_, err := io.Copy(&buffer, input)
if err != nil {
return nil, err
}
return streaming.NopCloser(bytes.NewReader(buffer.Bytes())), nil
}
// quoteETag wraps a raw Azure block id (used as a multipart part ETag) in
// double quotes. S3 ETags are quoted strings, and the posix backend follows
// the same convention (see backend.GenerateEtag). Keeping all Azure part-ETag
// surfaces (UploadPart, ListParts, UploadPartCopy) quoted ensures the values
// returned to clients are consistent, while the raw (unquoted) block id is
// still used for the Azure StageBlock/GetBlockList APIs.
func quoteETag(etag string) string {
return fmt.Sprintf("%q", etag)
}
// Creates a new Base64 encoded block id from a 32 bit integer
func blockIDInt32ToBase64(blockID int32) string {
binaryBlockID := &[4]byte{} // All block IDs are 4 bytes long
binary.LittleEndian.PutUint32(binaryBlockID[:], uint32(blockID))
return base64.StdEncoding.EncodeToString(binaryBlockID[:])
}
// Decodes Base64 encoded string to integer
func decodeBlockId(blockID string) (int, error) {
slice, err := base64.StdEncoding.DecodeString(blockID)
if err != nil {
return 0, nil
}
return int(binary.LittleEndian.Uint32(slice)), nil
}
func encodeBytes(b []byte) string {
return base64.StdEncoding.EncodeToString(b)
}
func decodeString(str string) ([]byte, error) {
if str == "" {
return []byte{}, nil
}
decoded, err := base64.StdEncoding.DecodeString(str)
if err != nil {
return nil, err
}
return decoded, nil
}
func (az *Azure) getContainerMetaData(ctx context.Context, bucket, key string) ([]byte, error) {
client, err := az.getContainerClient(bucket)
if err != nil {
return nil, err
}
props, err := client.GetProperties(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
if props.Metadata == nil {
return []byte{}, nil
}
data, ok := props.Metadata[key]
if !ok {
return []byte{}, nil
}
value, err := decodeString(*data)
if err != nil {
return nil, err
}
return value, nil
}
func (az *Azure) setContainerMetaData(ctx context.Context, bucket, key string, value []byte) error {
client, err := az.getContainerClient(bucket)
if err != nil {
return err
}
props, err := client.GetProperties(ctx, nil)
if err != nil {
return azureErrToS3Err(err)
}
mdmap := props.Metadata
if mdmap == nil {
mdmap = make(map[string]*string)
}
str := encodeBytes(value)
mdmap[key] = backend.GetPtrFromString(str)
_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: mdmap})
if err != nil {
return err
}
return nil
}
func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string) error {
client, err := az.getContainerClient(bucket)
if err != nil {
return err
}
props, err := client.GetProperties(ctx, nil)
if err != nil {
return azureErrToS3Err(err)
}
mdmap := props.Metadata
if mdmap == nil {
mdmap = make(map[string]*string)
}
delete(mdmap, key)
_, err = client.SetMetadata(ctx, &container.SetMetadataOptions{Metadata: mdmap})
if err != nil {
return err
}
return nil
}
func (az *Azure) evaluateWritePreconditions(ctx context.Context, bucket, object, ifMatch, ifNoneMatch *string) error {
if areNils(ifMatch, ifNoneMatch) {
return nil
}
// call HeadObject to evaluate preconditions
res, err := az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: bucket,
Key: object,
})
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
return err
}
var etag string
if res != nil {
etag = backend.GetStringFromPtr(res.ETag)
}
return backend.EvaluateObjectPutPreconditions(etag, ifMatch, ifNoneMatch, !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)))
}
func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
data, ok := meta[string(key)]
if !ok {
return &auth.ACL{}, nil
}
value, err := decodeString(*data)
if err != nil {
return nil, err
}
acl, err := auth.ParseACL(value)
if err != nil {
return nil, err
}
return &acl, nil
}
func createMetaTmpPath(obj, uploadId string) string {
objNameSum := sha256.Sum256([]byte(obj))
return filepath.Join(string(metaTmpMultipartPrefix), uploadId, fmt.Sprintf("%x", objNameSum))
}
// trackZeroBytePart records a zero-byte upload part in the sgwtmp metadata.
// Azure StageBlock rejects Content-Length: 0, so zero-byte parts are stored here.
func (az *Azure) trackZeroBytePart(ctx context.Context, bucket, key, uploadId string, partNumber int32) error {
tmpPath := createMetaTmpPath(key, uploadId)
blobClient, err := az.getBlobClient(bucket, tmpPath)
if err != nil {
return err
}
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
return azureErrToS3Err(err)
}
meta := props.Metadata
if meta == nil {
meta = map[string]*string{}
}
// Deduplicate: replace an existing entry for the same partNumber.
parts := parseZeroByteParts(meta)
found := slices.Contains(parts, partNumber)
if !found {
parts = append(parts, partNumber)
}
serialized := serializeZeroByteParts(parts)
meta[string(keyMpZeroBytesParts)] = &serialized
_, err = blobClient.SetMetadata(ctx, meta, nil)
return azureErrToS3Err(err)
}
// getZeroByteParts returns the list of zero-byte parts tracked in the sgwtmp metadata.
func (az *Azure) getZeroByteParts(ctx context.Context, bucket, key, uploadId string) ([]int32, error) {
tmpPath := createMetaTmpPath(key, uploadId)
blobClient, err := az.getBlobClient(bucket, tmpPath)
if err != nil {
return nil, err
}
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
return parseZeroByteParts(props.Metadata), nil
}
func parseZeroByteParts(meta map[string]*string) []int32 {
val, ok := meta[string(keyMpZeroBytesParts)]
if !ok || val == nil || *val == "" {
return nil
}
var parts []int32
for s := range strings.SplitSeq(*val, ",") {
n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 32)
if err == nil {
parts = append(parts, int32(n))
}
}
return parts
}
func serializeZeroByteParts(parts []int32) string {
strs := make([]string, len(parts))
for i, p := range parts {
strs[i] = strconv.Itoa(int(p))
}
return strings.Join(strs, ",")
}
// Checks if the multipart upload existis with the given bucket, key and uploadId
func (az *Azure) checkIfMpExists(ctx context.Context, bucket, obj, uploadId string) error {
tmpPath := createMetaTmpPath(obj, uploadId)
blobClient, err := az.getBlobClient(bucket, tmpPath)
if err != nil {
return err
}
_, err = blobClient.GetProperties(ctx, nil)
if err != nil {
return s3err.GetNoSuchUploadErr(uploadId)
}
return nil
}
func convertAzureEtag(etag *azcore.ETag) string {
// Azure ETag values are not S3 compatible,
// so append "-1" to avoid client SDK ETag validation issues.
str := (*string)(etag)
return *backend.TrimEtag(str) + "-1"
}
func areNils[T any](args ...*T) bool {
for _, arg := range args {
if arg != nil {
return false
}
}
return true
}
Reference in New Issue View Git Blame Copy Permalink