mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-27 02:14:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Merge conflicts resolved

Browse Source
This commit is contained in:
jonaustin09
2023-05-31 22:41:52 +04:00
parent ecd28bc2f7 c0cc170f78
commit f174308e3f
31 changed files with 981 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/auth/iam.go
+15 -1
View File
@@ -1,6 +1,20 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import "github.com/versity/scoutgw/s3err"
import "github.com/versity/versitygw/s3err"
type IAMConfig struct {
AccessAccounts map[string]string
backend/backend.go
+17 -3
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
@@ -6,7 +20,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/scoutgw/s3err"
"github.com/versity/versitygw/s3err"
)
//go:generate moq -out backend_moq_test.go . Backend
@@ -31,7 +45,7 @@ type Backend interface {
PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error)
PutObject(*s3.PutObjectInput) (string, error)
HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error)
HeadObject(bucket, object string) (*s3.HeadObjectOutput, error)
GetObject(bucket, object, acceptRange string, startOffset, length int64, writer io.Writer) (*s3.GetObjectOutput, error)
GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error)
@@ -126,7 +140,7 @@ func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjects
func (BackendUnsupported) GetObject(bucket, object, acceptRange string, startOffset, length int64, writer io.Writer) (*s3.GetObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error) {
func (BackendUnsupported) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) {
backend/backend_moq_test.go
+4 -10
View File
@@ -65,7 +65,7 @@ var _ Backend = &BackendMock{}
// HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) {
// panic("mock out the HeadBucket method")
// },
// HeadObjectFunc: func(bucket string, object string, etag string) (*s3.HeadObjectOutput, error) {
// HeadObjectFunc: func(bucket string, object string) (*s3.HeadObjectOutput, error) {
// panic("mock out the HeadObject method")
// },
// ListBucketsFunc: func() (*s3.ListBucketsOutput, error) {
@@ -172,7 +172,7 @@ type BackendMock struct {
HeadBucketFunc func(bucket string) (*s3.HeadBucketOutput, error)
// HeadObjectFunc mocks the HeadObject method.
HeadObjectFunc func(bucket string, object string, etag string) (*s3.HeadObjectOutput, error)
HeadObjectFunc func(bucket string, object string) (*s3.HeadObjectOutput, error)
// ListBucketsFunc mocks the ListBuckets method.
ListBucketsFunc func() (*s3.ListBucketsOutput, error)
@@ -350,8 +350,6 @@ type BackendMock struct {
Bucket string
// Object is the object argument value.
Object string
// Etag is the etag argument value.
Etag string
}
// ListBuckets holds details about calls to the ListBuckets method.
ListBuckets []struct {
@@ -1082,23 +1080,21 @@ func (mock *BackendMock) HeadBucketCalls() []struct {
}
// HeadObject calls HeadObjectFunc.
func (mock *BackendMock) HeadObject(bucket string, object string, etag string) (*s3.HeadObjectOutput, error) {
func (mock *BackendMock) HeadObject(bucket string, object string) (*s3.HeadObjectOutput, error) {
if mock.HeadObjectFunc == nil {
panic("BackendMock.HeadObjectFunc: method is nil but Backend.HeadObject was just called")
}
callInfo := struct {
Bucket string
Object string
Etag string
}{
Bucket: bucket,
Object: object,
Etag: etag,
}
mock.lockHeadObject.Lock()
mock.calls.HeadObject = append(mock.calls.HeadObject, callInfo)
mock.lockHeadObject.Unlock()
return mock.HeadObjectFunc(bucket, object, etag)
return mock.HeadObjectFunc(bucket, object)
}
// HeadObjectCalls gets all the calls that were made to HeadObject.
@@ -1108,12 +1104,10 @@ func (mock *BackendMock) HeadObject(bucket string, object string, etag string) (
func (mock *BackendMock) HeadObjectCalls() []struct {
Bucket string
Object string
Etag string
} {
var calls []struct {
Bucket string
Object string
Etag string
}
mock.lockHeadObject.RLock()
calls = mock.calls.HeadObject
backend/backend_test.go
+15 -1
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
@@ -7,7 +21,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/scoutgw/s3err"
"github.com/versity/versitygw/s3err"
)
func TestBackend_ListBuckets(t *testing.T) {
backend/common.go
+14
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
backend/posix/posix.go
+194 -87
View File
@@ -1,11 +1,28 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package posix
import (
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sort"
@@ -17,8 +34,8 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/uuid"
"github.com/pkg/xattr"
"github.com/versity/scoutgw/backend"
"github.com/versity/scoutgw/s3err"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
)
type Posix struct {
@@ -98,7 +115,7 @@ func (p *Posix) ListBuckets() (*s3.ListBucketsOutput, error) {
func (p *Posix) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
_, err := os.Lstat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -122,7 +139,7 @@ func (p *Posix) PutBucket(bucket string) error {
func (p *Posix) DeleteBucket(bucket string) error {
names, err := os.ReadDir(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -133,7 +150,7 @@ func (p *Posix) DeleteBucket(bucket string) error {
// if .sgwtmp is only item in directory
// then clean this up before trying to remove the bucket
err = os.RemoveAll(filepath.Join(bucket, metaTmpDir))
if err != nil && !os.IsNotExist(err) {
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove temp dir: %w", err)
}
}
@@ -154,7 +171,7 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C
object := *mpu.Key
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -202,7 +219,7 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C
func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -234,7 +251,7 @@ func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts [
}
}
f, err := openTmpFile(metaTmpDir, object, 0)
f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
if err != nil {
return nil, fmt.Errorf("open temp file: %w", err)
}
@@ -319,7 +336,7 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte,
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err := os.Stat(filepath.Join(objdir, uploadID))
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if err != nil {
@@ -328,7 +345,7 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte,
return sum, nil
}
func loadUserMetaData(path string, m map[string]string) (tag, contentType, contentEncoding string) {
func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
ents, err := xattr.List(path)
if err != nil || len(ents) == 0 {
return
@@ -348,16 +365,7 @@ func loadUserMetaData(path string, m map[string]string) (tag, contentType, conte
m[strings.TrimPrefix(e, "user.")] = string(b)
}
b, err := xattr.Get(path, "user."+tagHdr)
tag = string(b)
if err != nil {
tag = ""
}
if tag != "" {
m[tagHdr] = tag
}
b, err = xattr.Get(path, "user.content-type")
b, err := xattr.Get(path, "user.content-type")
contentType = string(b)
if err != nil {
contentType = ""
@@ -467,7 +475,7 @@ func (p *Posix) AbortMultipartUpload(mpu *s3.AbortMultipartUploadInput) error {
uploadID := *mpu.UploadId
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -495,7 +503,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.Lis
bucket := *mpu.Bucket
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -585,7 +593,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.Lis
func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -600,7 +608,7 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
ents, err := os.ReadDir(filepath.Join(objdir, uploadID))
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if err != nil {
@@ -670,7 +678,7 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke
func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return "", s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -678,10 +686,11 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length
}
sum := sha256.Sum256([]byte(object))
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part))
f, err := openTmpFile(objdir, partPath, length)
f, err := openTmpFile(filepath.Join(bucket, objdir),
bucket, partPath, length)
if err != nil {
return "", fmt.Errorf("open temp file: %w", err)
}
@@ -708,7 +717,7 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length
func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
_, err := os.Stat(*po.Bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return "", s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -717,8 +726,6 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
name := filepath.Join(*po.Bucket, *po.Key)
etag := ""
if strings.HasSuffix(*po.Key, "/") {
// object is directory
err = mkdirAll(name, os.FileMode(0755), *po.Bucket, *po.Key)
@@ -730,50 +737,54 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
xattr.Set(name, "user."+k, []byte(v))
}
// set our tag that this dir was specifically put
// set our attribute that this dir was specifically put
xattr.Set(name, dirObjKey, nil)
} else {
// object is file
f, err := openTmpFile(metaTmpDir, *po.Key, po.ContentLength)
// TODO: what etag should be returned here
// and we should set etag xattr to identify dir was
// specifically uploaded
return "", nil
}
// object is file
f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
*po.Bucket, *po.Key, po.ContentLength)
if err != nil {
return "", fmt.Errorf("open temp file: %w", err)
}
defer f.cleanup()
hash := md5.New()
rdr := io.TeeReader(po.Body, hash)
_, err = io.Copy(f, rdr)
if err != nil {
return "", fmt.Errorf("write object data: %w", err)
}
dir := filepath.Dir(name)
if dir != "" {
err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key)
if err != nil {
return "", fmt.Errorf("open temp file: %w", err)
return "", fmt.Errorf("make object parent directories: %w", err)
}
defer f.cleanup()
}
// TODO: fallocate based on content length
err = f.link()
if err != nil {
return "", fmt.Errorf("link object in namespace: %w", err)
}
hash := md5.New()
rdr := io.TeeReader(po.Body, hash)
_, err = io.Copy(f, rdr)
for k, v := range po.Metadata {
xattr.Set(name, "user."+k, []byte(v))
}
dataSum := hash.Sum(nil)
etag := hex.EncodeToString(dataSum[:])
xattr.Set(name, "user.etag", []byte(etag))
if newObjUID != 0 || newObjGID != 0 {
err = os.Chown(name, newObjUID, newObjGID)
if err != nil {
return "", fmt.Errorf("write object data: %w", err)
}
dir := filepath.Dir(name)
if dir != "" {
err = mkdirAll(dir, os.FileMode(0755), *po.Bucket, *po.Key)
if err != nil {
return "", fmt.Errorf("make object parent directories: %w", err)
}
}
err = f.link()
if err != nil {
return "", fmt.Errorf("link object in namespace: %w", err)
}
for k, v := range po.Metadata {
xattr.Set(name, "user."+k, []byte(v))
}
dataSum := hash.Sum(nil)
etag := hex.EncodeToString(dataSum[:])
xattr.Set(name, "user.etag", []byte(etag))
if newObjUID != 0 || newObjGID != 0 {
err = os.Chown(name, newObjUID, newObjGID)
if err != nil {
return "", fmt.Errorf("set object uid/gid: %v", err)
}
return "", fmt.Errorf("set object uid/gid: %v", err)
}
}
@@ -782,7 +793,7 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
func (p *Posix) DeleteObject(bucket, object string) error {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -790,7 +801,7 @@ func (p *Posix) DeleteObject(bucket, object string) error {
}
os.Remove(filepath.Join(bucket, object))
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -844,7 +855,7 @@ func (p *Posix) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) err
func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, length int64, writer io.Writer) (*s3.GetObjectOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -853,7 +864,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -866,7 +877,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
}
f, err := os.Open(objPath)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -882,7 +893,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
userMetaData := make(map[string]string)
_, contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
b, err := xattr.Get(objPath, "user.etag")
etag := string(b)
@@ -890,8 +901,11 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
etag = ""
}
// TODO: fill range request header?
// TODO: parse tags for tag count?
tags, err := p.getXattrTags(bucket, object)
if err != nil {
return nil, fmt.Errorf("get object tags: %w", err)
}
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: length,
@@ -900,12 +914,13 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: int32(len(tags)),
}, nil
}
func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error) {
func (p *Posix) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -914,7 +929,7 @@ func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOu
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -922,10 +937,14 @@ func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOu
}
userMetaData := make(map[string]string)
_, contentType, contentEncoding := loadUserMetaData(filepath.Join(bucket, objPath), userMetaData)
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
b, err := xattr.Get(objPath, "user.etag")
etag := string(b)
if err != nil {
etag = ""
}
// TODO: fill accept ranges request header?
// TODO: do we need to get etag from xattr?
return &s3.HeadObjectOutput{
ContentLength: fi.Size(),
ContentType: &contentType,
@@ -938,7 +957,7 @@ func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOu
func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
_, err := os.Stat(srcBucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -946,7 +965,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
}
_, err = os.Stat(DstBucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -955,7 +974,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
objPath := filepath.Join(srcBucket, srcObject)
f, err := os.Open(objPath)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -983,7 +1002,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -1008,9 +1027,10 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (
Prefix: &prefix,
}, nil
}
func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -1035,3 +1055,90 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int)
Prefix: &prefix,
}, nil
}
func (p *Posix) GetTags(bucket, object string) (map[string]string, error) {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
return p.getXattrTags(bucket, object)
}
func (p *Posix) getXattrTags(bucket, object string) (map[string]string, error) {
tags := make(map[string]string)
b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if isNoAttr(err) {
return tags, nil
}
if err != nil {
return nil, fmt.Errorf("get tags: %w", err)
}
err = json.Unmarshal(b, &tags)
if err != nil {
return nil, fmt.Errorf("unmarshal tags: %w", err)
}
return tags, nil
}
func (p *Posix) SetTags(bucket, object string, tags map[string]string) error {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return fmt.Errorf("stat bucket: %w", err)
}
if tags == nil {
err = xattr.Remove(filepath.Join(bucket, object), "user."+tagHdr)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return fmt.Errorf("remove tags: %w", err)
}
return nil
}
b, err := json.Marshal(tags)
if err != nil {
return fmt.Errorf("marshal tags: %w", err)
}
err = xattr.Set(filepath.Join(bucket, object), "user."+tagHdr, b)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return fmt.Errorf("set tags: %w", err)
}
return nil
}
func (p *Posix) RemoveTags(bucket, object string) error {
return p.SetTags(bucket, object, nil)
}
func isNoAttr(err error) bool {
if err == nil {
return false
}
xerr, ok := err.(*xattr.Error)
if ok && xerr.Err == xattr.ENOATTR {
return true
}
if err == syscall.ENODATA {
return true
}
return false
}
backend/posix/posix_darwin.go
+39 -6
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package posix
import (
@@ -6,30 +20,43 @@ import (
"fmt"
"io/fs"
"os"
"path/filepath"
)
type tmpfile struct {
f *os.File
bucket string
objname string
size int64
}
func openTmpFile(dir, obj string, size int64) (*tmpfile, error) {
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
// Create a temp file for upload while in progress (see link comments below).
err := os.MkdirAll(dir, 0700)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x\n", sha256.Sum256([]byte(obj))))
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
return nil, err
}
return &tmpfile{f: f, objname: obj}, nil
return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
}
func (tmp *tmpfile) link() error {
tempname := tmp.f.Name()
// cleanup in case anything goes wrong, if rename succeeds then
// this will no longer exist
defer os.Remove(tempname)
// We use Rename as the atomic operation for object puts. The upload is
// written to a temp file to not conflict with any other simultaneous
// uploads. The final operation is to move the temp file into place for
// the object. This ensures the object semantics of last upload completed
// wins and is not some combination of writes from simultaneous uploads.
err := os.Remove(tmp.objname)
objPath := filepath.Join(tmp.bucket, tmp.objname)
err := os.Remove(objPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove stale path: %w", err)
}
@@ -39,7 +66,7 @@ func (tmp *tmpfile) link() error {
return fmt.Errorf("close tmpfile: %w", err)
}
err = os.Rename(tmp.f.Name(), tmp.objname)
err = os.Rename(tempname, objPath)
if err != nil {
return fmt.Errorf("rename tmpfile: %w", err)
}
@@ -48,7 +75,13 @@ func (tmp *tmpfile) link() error {
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
return tmp.f.Write(b)
if int64(len(b)) > tmp.size {
return 0, fmt.Errorf("write exceeds content length")
}
n, err := tmp.f.Write(b)
tmp.size -= int64(n)
return n, err
}
func (tmp *tmpfile) cleanup() {
backend/posix/posix_linux.go
+74 -33
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package posix
import (
@@ -17,11 +31,13 @@ const procfddir = "/proc/self/fd"
type tmpfile struct {
f *os.File
bucket string
objname string
isOTmp bool
size int64
}
func openTmpFile(dir, obj string, size int64) (*tmpfile, error) {
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
// This can help reduce contention within the namespace (parent directories),
// etc. And will auto cleanup the inode on close if we never link this
@@ -31,30 +47,37 @@ func openTmpFile(dir, obj string, size int64) (*tmpfile, error) {
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
if err != nil {
// O_TMPFILE not supported, try fallback
err := os.MkdirAll(dir, 0700)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x\n", sha256.Sum256([]byte(obj))))
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
return nil, err
}
tmp := &tmpfile{f: f}
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
// falloc is best effort, its fine if this fails
if size > 0 {
tmp.falloc(size)
tmp.falloc()
}
return tmp, nil
}
// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
// later to link file into namespace
f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
tmp := &tmpfile{f: f, isOTmp: true}
tmp := &tmpfile{f: f, isOTmp: true, size: size}
// falloc is best effort, its fine if this fails
if size > 0 {
tmp.falloc(size)
tmp.falloc()
}
return tmp, nil
}
func (tmp *tmpfile) falloc(size int64) error {
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, size)
func (tmp *tmpfile) falloc() error {
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
if err != nil {
return fmt.Errorf("fallocate: %v", err)
}
@@ -68,36 +91,33 @@ func (tmp *tmpfile) link() error {
// temp file into place for the object. This ensures the object semantics
// of last upload completed wins and is not some combination of writes
// from simultaneous uploads.
err := os.Remove(tmp.objname)
objPath := filepath.Join(tmp.bucket, tmp.objname)
err := os.Remove(objPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove stale path: %w", err)
}
if tmp.isOTmp {
procdir, err := os.Open(procfddir)
if err != nil {
return fmt.Errorf("open proc dir: %w", err)
}
defer procdir.Close()
if !tmp.isOTmp {
// O_TMPFILE not suported, use fallback
return tmp.fallbackLink()
}
dir, err := os.Open(filepath.Dir(tmp.objname))
if err != nil {
return fmt.Errorf("open parent dir: %w", err)
}
defer dir.Close()
procdir, err := os.Open(procfddir)
if err != nil {
return fmt.Errorf("open proc dir: %w", err)
}
defer procdir.Close()
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dir.Fd()), filepath.Base(tmp.objname), unix.AT_SYMLINK_FOLLOW)
if err != nil {
return fmt.Errorf("link tmpfile: %w", err)
}
dir, err := os.Open(filepath.Dir(objPath))
if err != nil {
return fmt.Errorf("open parent dir: %w", err)
}
defer dir.Close()
err = tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
return nil
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
if err != nil {
return fmt.Errorf("link tmpfile: %w", err)
}
err = tmp.f.Close()
@@ -105,7 +125,22 @@ func (tmp *tmpfile) link() error {
return fmt.Errorf("close tmpfile: %w", err)
}
err = os.Rename(tmp.f.Name(), tmp.objname)
return nil
}
func (tmp *tmpfile) fallbackLink() error {
tempname := tmp.f.Name()
// cleanup in case anything goes wrong, if rename succeeds then
// this will no longer exist
defer os.Remove(tempname)
err := tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
objPath := filepath.Join(tmp.bucket, tmp.objname)
err = os.Rename(tempname, objPath)
if err != nil {
return fmt.Errorf("rename tmpfile: %w", err)
}
@@ -114,7 +149,13 @@ func (tmp *tmpfile) link() error {
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
return tmp.f.Write(b)
if int64(len(b)) > tmp.size {
return 0, fmt.Errorf("write exceeds content length")
}
n, err := tmp.f.Write(b)
tmp.size -= int64(n)
return n, err
}
func (tmp *tmpfile) cleanup() {
backend/scoutfs/scoutfs.go
+16 -2
View File
@@ -1,8 +1,22 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package scoutfs
import (
"github.com/versity/scoutgw/backend"
"github.com/versity/scoutgw/backend/posix"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/backend/posix"
)
type ScoutFS struct {
backend/walk.go
+14
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
backend/walk_test.go
+15 -1
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend_test
import (
@@ -6,7 +20,7 @@ import (
"testing/fstest"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/scoutgw/backend"
"github.com/versity/versitygw/backend"
)
type walkTest struct {