diff --git a/backend/azure/azure.go b/backend/azure/azure.go new file mode 100644 index 00000000..17f29ccf --- /dev/null +++ b/backend/azure/azure.go @@ -0,0 +1,733 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package azure + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +type Azure struct { + backend.BackendUnsupported + + client *azblob.Client + creds *azblob.SharedKeyCredential + serviceURL string +} + +var _ backend.Backend = &Azure{} + +func New(accountName, accountKey, serviceURL string) (*Azure, error) { + cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + return nil, fmt.Errorf("init credentials: %w", err) + } + + client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) + if err != nil { + return nil, fmt.Errorf("init client: %w", err) + } + + return &Azure{client: client, serviceURL: serviceURL, creds: cred}, nil +} + +func (az *Azure) Shutdown() {} + +func (az *Azure) String() string { + return "Azure Blob Gateway" +} + +func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error { + // TODO: handle ownership/ACLs + _, err := az.client.CreateContainer(ctx, *input.Bucket, nil) + return azureErrToS3Err(err) +} + +func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) { + pager := az.client.NewListContainersPager(nil) + + var buckets []s3response.ListAllMyBucketsEntry + var result s3response.ListAllMyBucketsResult + + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return result, azureErrToS3Err(err) + } + for _, v := range resp.ContainerItems { + buckets = append(buckets, s3response.ListAllMyBucketsEntry{ + Name: *v.Name, + // TODO: using modification date here instead of creation, is that ok? + CreationDate: *v.Properties.LastModified, + }) + } + } + + result.Buckets.Bucket = buckets + + return result, nil +} + +func (az *Azure) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { + client, err := az.getContainerClient(*input.Bucket) + if err != nil { + return nil, err + } + + _, err = client.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + return &s3.HeadBucketOutput{}, nil +} + +func (az *Azure) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error { + _, err := az.client.DeleteContainer(ctx, *input.Bucket, nil) + return azureErrToS3Err(err) +} + +func (az *Azure) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) { + tags, err := parseTags(po.Tagging) + if err != nil { + return "", err + } + + uploadResp, err := az.client.UploadStream(ctx, *po.Bucket, *po.Key, po.Body, &blockblob.UploadStreamOptions{ + Metadata: parseMetadata(po.Metadata), + Tags: tags, + }) + if err != nil { + return "", azureErrToS3Err(err) + } + + return string(*uploadResp.ETag), nil +} + +func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) { + var opts *azblob.DownloadStreamOptions + if input.Range != nil { + offset, count, err := parseRange(*input.Range) + if err != nil { + return nil, err + } + opts = &azblob.DownloadStreamOptions{ + Range: blob.HTTPRange{ + Count: count, + Offset: offset, + }, + } + } + blobDownloadResponse, err := az.client.DownloadStream(ctx, *input.Bucket, *input.Key, opts) + if err != nil { + return nil, azureErrToS3Err(err) + } + defer blobDownloadResponse.Body.Close() + + _, err = io.Copy(writer, blobDownloadResponse.Body) + if err != nil { + return nil, fmt.Errorf("copy data: %w", err) + } + + var tagcount int32 + if blobDownloadResponse.TagCount != nil { + tagcount = int32(*blobDownloadResponse.TagCount) + } + + return &s3.GetObjectOutput{ + AcceptRanges: blobDownloadResponse.AcceptRanges, + ContentLength: blobDownloadResponse.ContentLength, + ContentEncoding: blobDownloadResponse.ContentEncoding, + ContentType: blobDownloadResponse.ContentType, + ETag: (*string)(blobDownloadResponse.ETag), + LastModified: blobDownloadResponse.LastModified, + Metadata: parseAzMetadata(blobDownloadResponse.Metadata), + TagCount: &tagcount, + ContentRange: blobDownloadResponse.ContentRange, + }, nil +} + +func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + client, err := az.getBlobClient(*input.Bucket, *input.Key) + if err != nil { + return nil, err + } + + resp, err := client.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + return &s3.HeadObjectOutput{ + AcceptRanges: resp.AcceptRanges, + ContentLength: resp.ContentLength, + ContentType: resp.ContentType, + ContentEncoding: resp.ContentEncoding, + ContentLanguage: resp.ContentLanguage, + ContentDisposition: resp.ContentDisposition, + ETag: (*string)(resp.ETag), + LastModified: resp.LastModified, + Metadata: parseAzMetadata(resp.Metadata), + Expires: resp.ExpiresOn, + }, nil +} + +func (az *Azure) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { + pager := az.client.NewListBlobsFlatPager(*input.Bucket, &azblob.ListBlobsFlatOptions{ + Marker: input.Marker, + MaxResults: input.MaxKeys, + Prefix: input.Prefix, + }) + + var objects []types.Object + var nextMarker *string + + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, azureErrToS3Err(err) + } + for _, v := range resp.Segment.BlobItems { + nextMarker = resp.NextMarker + objects = append(objects, types.Object{ + ETag: (*string)(v.Properties.ETag), + Key: v.Name, + LastModified: v.Properties.LastModified, + Size: v.Properties.ContentLength, + StorageClass: types.ObjectStorageClass(*v.Properties.AccessTier), + }) + } + } + + // TODO: generate common prefixes when appropriate + // TODO: set truncated response status + + return &s3.ListObjectsOutput{ + Contents: objects, + Marker: input.Marker, + MaxKeys: input.MaxKeys, + Name: input.Bucket, + NextMarker: nextMarker, + Prefix: input.Prefix, + }, nil +} + +func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + pager := az.client.NewListBlobsFlatPager(*input.Bucket, &azblob.ListBlobsFlatOptions{ + Marker: input.ContinuationToken, + MaxResults: input.MaxKeys, + Prefix: input.Prefix, + }) + + var objects []types.Object + var nextMarker *string + + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, azureErrToS3Err(err) + } + for _, v := range resp.Segment.BlobItems { + nextMarker = resp.NextMarker + objects = append(objects, types.Object{ + ETag: (*string)(v.Properties.ETag), + Key: v.Name, + LastModified: v.Properties.LastModified, + Size: v.Properties.ContentLength, + StorageClass: types.ObjectStorageClass(*v.Properties.AccessTier), + }) + } + } + + // TODO: generate common prefixes when appropriate + // TODO: set truncated response status + + return &s3.ListObjectsV2Output{ + Contents: objects, + ContinuationToken: input.ContinuationToken, + MaxKeys: input.MaxKeys, + Name: input.Bucket, + NextContinuationToken: nextMarker, + Prefix: input.Prefix, + }, nil +} + +func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error { + _, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) + return azureErrToS3Err(err) +} + +func (az *Azure) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) { + delResult, errs := []types.DeletedObject{}, []types.Error{} + for _, obj := range input.Delete.Objects { + err := az.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: input.Bucket, + Key: obj.Key, + }) + if err == nil { + delResult = append(delResult, types.DeletedObject{Key: obj.Key}) + } else { + serr, ok := err.(s3err.APIError) + if ok { + errs = append(errs, types.Error{ + Key: obj.Key, + Code: &serr.Code, + Message: &serr.Description, + }) + } else { + errs = append(errs, types.Error{ + Key: obj.Key, + Code: getStringPtr("InternalError"), + Message: getStringPtr(err.Error()), + }) + } + } + } + + return s3response.DeleteObjectsResult{ + Deleted: delResult, + Error: errs, + }, nil +} + +func (az *Azure) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { + client, err := az.getBlobClient(*input.Bucket, *input.Key) + if err != nil { + return nil, err + } + + tags, err := parseTags(input.Tagging) + if err != nil { + return nil, err + } + + resp, err := client.CopyFromURL(ctx, az.serviceURL+"/"+*input.CopySource, &blob.CopyFromURLOptions{ + BlobTags: tags, + Metadata: parseMetadata(input.Metadata), + }) + if err != nil { + return nil, azureErrToS3Err(err) + } + + return &s3.CopyObjectOutput{ + CopyObjectResult: &types.CopyObjectResult{ + ETag: (*string)(resp.ETag), + LastModified: resp.LastModified, + }, + }, nil +} + +func (az *Azure) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error { + client, err := az.getBlobClient(bucket, object) + if err != nil { + return err + } + + _, err = client.SetTags(ctx, tags, nil) + if err != nil { + return azureErrToS3Err(err) + } + + return nil +} + +func (az *Azure) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) { + client, err := az.getBlobClient(bucket, object) + if err != nil { + return nil, err + } + + tags, err := client.GetTags(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + return parseAzTags(tags.BlobTagSet), nil +} + +func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object string) error { + client, err := az.getBlobClient(bucket, object) + if err != nil { + return err + } + + //TODO: SDK has a bug here: it recommends to use the method to remove tags by passing an empty map, + // but the method panics because of incorrect implementation + _, err = client.SetTags(ctx, map[string]string{}, nil) + if err != nil { + return azureErrToS3Err(err) + } + + return nil +} + +// Multipart upload starts with UploadPart action. +// Each part is translated into an uncommitted block in a newly created blob in staging area +func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) { + client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) + if err != nil { + return "", err + } + + rdr, err := getReadSeekCloser(input.Body) + if err != nil { + return "", err + } + + // block id serves as etag here + etag = blockIDInt32ToBase64(*input.PartNumber) + _, err = client.StageBlock(ctx, etag, rdr, nil) + if err != nil { + return "", azureErrToS3Err(err) + } + + return etag, nil +} + +func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { + client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) + if err != nil { + return s3response.CopyObjectResult{}, nil + } + + //TODO: handle block copy by range + //TODO: the action returns not implemented on azurite, maybe in production this will work? + // UploadId here is the source block id + _, err = client.StageBlockFromURL(ctx, *input.UploadId, *input.CopySource, nil) + if err != nil { + return s3response.CopyObjectResult{}, azureErrToS3Err(err) + } + + return s3response.CopyObjectResult{}, nil +} + +// Lists all uncommitted parts from the blob +func (az *Azure) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) { + client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) + if err != nil { + return s3response.ListPartsResult{}, nil + } + + resp, err := client.GetBlockList(ctx, blockblob.BlockListTypeUncommitted, nil) + if err != nil { + return s3response.ListPartsResult{}, azureErrToS3Err(err) + } + var partNumberMarker int + var nextPartNumberMarker int + var maxParts int32 = math.MaxInt32 + var isTruncated bool + + if *input.PartNumberMarker != "" { + partNumberMarker, err = strconv.Atoi(*input.PartNumberMarker) + if err != nil { + return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker) + } + } + if input.MaxParts != nil { + maxParts = *input.MaxParts + } + + parts := []s3response.Part{} + for _, el := range resp.BlockList.UncommittedBlocks { + partNumber, err := decodeBlockId(*el.Name) + if err != nil { + return s3response.ListPartsResult{}, err + } + if partNumberMarker != 0 && partNumberMarker < partNumber { + continue + } + if len(parts) >= int(maxParts) { + nextPartNumberMarker = partNumber + isTruncated = true + break + } + parts = append(parts, s3response.Part{ + Size: *el.Size, + ETag: *el.Name, + PartNumber: partNumber, + }) + } + return s3response.ListPartsResult{ + Bucket: *input.Bucket, + Key: *input.Key, + Parts: parts, + NextPartNumberMarker: nextPartNumberMarker, + PartNumberMarker: partNumberMarker, + IsTruncated: isTruncated, + MaxParts: int(maxParts), + }, nil +} + +// Lists all block blobs, which has uncommitted blocks +func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + client, err := az.getContainerClient(*input.Bucket) + if err != nil { + return s3response.ListMultipartUploadsResult{}, nil + } + pager := client.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Include: container.ListBlobsInclude{UncommittedBlobs: true}, + Marker: input.KeyMarker, + Prefix: input.Prefix, + }) + + var maxUploads int32 + if input.MaxUploads != nil { + maxUploads = *input.MaxUploads + } + isTruncated := false + nextKeyMarker := "" + uploads := []s3response.Upload{} + breakFlag := false + + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return s3response.ListMultipartUploadsResult{}, azureErrToS3Err(err) + } + for _, el := range resp.Segment.BlobItems { + if el.Properties.AccessTier == nil { + if len(uploads) >= int(*input.MaxUploads) && maxUploads != 0 { + breakFlag = true + nextKeyMarker = *el.Name + isTruncated = true + break + } + uploads = append(uploads, s3response.Upload{ + Key: *el.Name, + Initiated: el.Properties.CreationTime.Format(backend.RFC3339TimeFormat), + }) + } + } + if breakFlag { + break + } + } + return s3response.ListMultipartUploadsResult{ + Uploads: uploads, + Bucket: *input.Bucket, + KeyMarker: *input.KeyMarker, + NextKeyMarker: nextKeyMarker, + MaxUploads: int(maxUploads), + Prefix: *input.Prefix, + IsTruncated: isTruncated, + Delimiter: *input.Delimiter, + }, nil +} + +// Deletes the block blob with committed/uncommitted blocks +func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { + _, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) + if err != nil { + return azureErrToS3Err(err) + } + return nil +} + +// Commits all the uncommitted blocks inside the block blob +// And moves the block blob from staging area into the blobs list +// It indicates the end of the multipart upload +func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { + client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) + if err != nil { + return nil, err + } + blockIds := []string{} + for _, el := range input.MultipartUpload.Parts { + blockIds = append(blockIds, *el.ETag) + } + resp, err := client.CommitBlockList(ctx, blockIds, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + return &s3.CompleteMultipartUploadOutput{ + Bucket: input.Bucket, + Key: input.Key, + ETag: (*string)(resp.ETag), + }, nil +} + +func (az *Azure) getBlobClient(container, blb string) (*blob.Client, error) { + return blob.NewClientWithSharedKeyCredential(fmt.Sprintf("%v/%v/%v", az.serviceURL, container, blb), az.creds, nil) +} + +func (az *Azure) getContainerClient(ctr string) (*container.Client, error) { + return container.NewClientWithSharedKeyCredential(fmt.Sprintf("%v/%v", az.serviceURL, ctr), az.creds, nil) +} + +func (az *Azure) getBlockBlobClient(container, blob string) (*blockblob.Client, error) { + return blockblob.NewClientWithSharedKeyCredential(fmt.Sprintf("%v/%v/%v", az.serviceURL, container, blob), az.creds, nil) +} + +func parseMetadata(m map[string]string) map[string]*string { + if m == nil { + return nil + } + + meta := make(map[string]*string) + + for k, v := range m { + meta[k] = &v + } + return meta +} + +func parseAzMetadata(m map[string]*string) map[string]string { + if m == nil { + return nil + } + + meta := make(map[string]string) + + for k, v := range m { + meta[k] = *v + } + return meta +} + +func parseTags(tagstr *string) (map[string]string, error) { + tagsStr := getString(tagstr) + tags := make(map[string]string) + + if tagsStr != "" { + tagParts := strings.Split(tagsStr, "&") + for _, prt := range tagParts { + p := strings.Split(prt, "=") + if len(p) != 2 { + return nil, s3err.GetAPIError(s3err.ErrInvalidTag) + } + tags[p[0]] = p[1] + } + } + + return tags, nil +} + +func parseAzTags(tagSet []*blob.Tags) map[string]string { + tags := map[string]string{} + for _, tag := range tagSet { + tags[*tag.Key] = *tag.Value + } + + return tags +} + +func getString(str *string) string { + if str == nil { + return "" + } + return *str +} + +func getStringPtr(str string) *string { + return &str +} + +// Parses azure ResponseError into AWS APIError +func azureErrToS3Err(apiErr error) error { + var azErr *azcore.ResponseError + if !errors.As(apiErr, &azErr) { + return apiErr + } + + resp := s3err.APIError{ + Code: azErr.ErrorCode, + Description: azErr.RawResponse.Status, + HTTPStatusCode: azErr.StatusCode, + } + fmt.Println(resp) + return resp +} + +// Converts io.Reader into io.ReadSeekCloser +func getReadSeekCloser(input io.Reader) (io.ReadSeekCloser, error) { + var buffer bytes.Buffer + _, err := io.Copy(&buffer, input) + if err != nil { + return nil, err + } + + return streaming.NopCloser(bytes.NewReader(buffer.Bytes())), nil +} + +// Creates a new Base64 encoded block id from a 32 bit integer +func blockIDInt32ToBase64(blockID int32) string { + binaryBlockID := &[4]byte{} // All block IDs are 4 bytes long + binary.LittleEndian.PutUint32(binaryBlockID[:], uint32(blockID)) + return base64.StdEncoding.EncodeToString(binaryBlockID[:]) +} + +// Decodes Base64 encoded string to integer +func decodeBlockId(blockID string) (int, error) { + slice, err := base64.StdEncoding.DecodeString(blockID) + if err != nil { + return 0, nil + } + + return int(binary.LittleEndian.Uint32(slice)), nil +} + +func parseRange(rg string) (offset, count int64, err error) { + rangeKv := strings.Split(rg, "=") + + if len(rangeKv) < 2 { + return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange) + } + + bRange := strings.Split(rangeKv[1], "-") + if len(bRange) < 1 || len(bRange) > 2 { + return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange) + } + + offset, err = strconv.ParseInt(bRange[0], 10, 64) + if err != nil { + return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange) + } + + if len(bRange) == 1 || bRange[1] == "" { + return offset, count, nil + } + + count, err = strconv.ParseInt(bRange[1], 10, 64) + if err != nil { + return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange) + } + + if count < offset { + return 0, 0, s3err.GetAPIError(s3err.ErrInvalidRange) + } + + return offset, count - offset + 1, nil +} diff --git a/cmd/versitygw/azure.go b/cmd/versitygw/azure.go new file mode 100644 index 00000000..8fe81f4d --- /dev/null +++ b/cmd/versitygw/azure.go @@ -0,0 +1,72 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "fmt" + + "github.com/urfave/cli/v2" + "github.com/versity/versitygw/backend/azure" +) + +var ( + azAccount, azKey, azServiceURL string +) + +func azureCommand() *cli.Command { + return &cli.Command{ + Name: "azure", + Usage: "azure blob storage backend", + Description: `direct translation from s3 objects to azure blobs`, + Action: runAzure, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "account", + Usage: "azure account name", + EnvVars: []string{"AZ_ACCESS_KEY"}, + Aliases: []string{"a"}, + Destination: &azAccount, + }, + &cli.StringFlag{ + Name: "account-key", + Usage: "azure account key", + EnvVars: []string{"AZ_ACCOUNT_KEY"}, + Aliases: []string{"s"}, + Destination: &azKey, + }, + &cli.StringFlag{ + Name: "url", + Usage: "azure service URL", + EnvVars: []string{"AZ_ENDPOINT"}, + Aliases: []string{"u"}, + Destination: &azServiceURL, + }, + }, + } +} + +func runAzure(ctx *cli.Context) error { + if azServiceURL == "" { + // if not otherwise specified, use the typical form: http(s)://.blob.core.windows.net/ + azServiceURL = fmt.Sprintf("https://%s.blob.core.windows.net/", azAccount) + } + + be, err := azure.New(azAccount, azKey, azServiceURL) + if err != nil { + return fmt.Errorf("init azure: %v", err) + } + + return runGateway(ctx.Context, be) +} diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index f99faed4..28321f56 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -75,6 +75,7 @@ func main() { posixCommand(), scoutfsCommand(), s3Command(), + azureCommand(), adminCommand(), testCommand(), } diff --git a/go.mod b/go.mod index 2aa6b71f..b94e3c54 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/versity/versitygw go 1.20 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 github.com/aws/aws-sdk-go-v2 v1.24.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7 github.com/aws/smithy-go v1.19.0 @@ -19,6 +21,7 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect @@ -30,8 +33,9 @@ require ( github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect - github.com/stretchr/testify v1.8.1 // indirect golang.org/x/crypto v0.17.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect ) require ( diff --git a/go.sum b/go.sum index 03cc663d..b80dfe04 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,14 @@ +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= @@ -47,12 +56,14 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A= github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ= github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= @@ -64,6 +75,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -80,6 +92,7 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -93,11 +106,9 @@ github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUan github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -130,8 +141,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -163,14 +175,15 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/s3response/s3response.go b/s3response/s3response.go index 07d3fa6e..cb0b401f 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -73,7 +73,7 @@ type ListMultipartUploadsResult struct { CommonPrefixes []CommonPrefix } -// Upload desribes in progress multipart upload +// Upload describes in progress multipart upload type Upload struct { Key string UploadID string `xml:"UploadId"`