mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-02-11 20:51:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #25 from versity/ben/posix

Browse Source 
Ben/posix
This commit is contained in:
Ben McClelland
2023-05-25 10:17:16 -07:00
committed by GitHub
parent 355e99a7ef cc1fb2cffe
commit aa00a89e5c
1 changed files with 114 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
backend/posix/posix.go
View File

@@ -4,8 +4,11 @@ import (
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sort"
@@ -98,7 +101,7 @@ func (p *Posix) ListBuckets() (*s3.ListBucketsOutput, error) {
func (p *Posix) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
_, err := os.Lstat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -122,7 +125,7 @@ func (p *Posix) PutBucket(bucket string) error {
func (p *Posix) DeleteBucket(bucket string) error {
names, err := os.ReadDir(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -133,7 +136,7 @@ func (p *Posix) DeleteBucket(bucket string) error {
// if .sgwtmp is only item in directory
// then clean this up before trying to remove the bucket
err = os.RemoveAll(filepath.Join(bucket, metaTmpDir))
if err != nil && !os.IsNotExist(err) {
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove temp dir: %w", err)
}
}
@@ -154,7 +157,7 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C
object := *mpu.Key
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -202,7 +205,7 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C
func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -319,7 +322,7 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte,
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err := os.Stat(filepath.Join(objdir, uploadID))
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if err != nil {
@@ -467,7 +470,7 @@ func (p *Posix) AbortMultipartUpload(mpu *s3.AbortMultipartUploadInput) error {
uploadID := *mpu.UploadId
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -495,7 +498,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.Lis
bucket := *mpu.Bucket
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -585,7 +588,7 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (*s3.Lis
func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -600,7 +603,7 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
ents, err := os.ReadDir(filepath.Join(objdir, uploadID))
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if err != nil {
@@ -670,7 +673,7 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke
func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return "", s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -708,7 +711,7 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length
func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
_, err := os.Stat(*po.Bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return "", s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -782,7 +785,7 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) {
func (p *Posix) DeleteObject(bucket, object string) error {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -790,7 +793,7 @@ func (p *Posix) DeleteObject(bucket, object string) error {
}
os.Remove(filepath.Join(bucket, object))
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -844,7 +847,7 @@ func (p *Posix) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) err
func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, length int64, writer io.Writer) (*s3.GetObjectOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -853,7 +856,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -866,7 +869,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
}
f, err := os.Open(objPath)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -890,8 +893,11 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
etag = ""
}
// TODO: fill range request header?
// TODO: parse tags for tag count?
tags, err := p.getXattrTags(bucket, object)
if err != nil {
return nil, fmt.Errorf("get object tags: %w", err)
}
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: length,
@@ -900,12 +906,13 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, startOffset, lengt
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: int32(len(tags)),
}, nil
}
func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -914,7 +921,7 @@ func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOu
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -938,7 +945,7 @@ func (p *Posix) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOu
func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
_, err := os.Stat(srcBucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -946,7 +953,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
}
_, err = os.Stat(DstBucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -955,7 +962,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
objPath := filepath.Join(srcBucket, srcObject)
f, err := os.Open(objPath)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -983,7 +990,7 @@ func (p *Posix) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*
func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -1008,9 +1015,10 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (
Prefix: &prefix,
}, nil
}
func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
_, err := os.Stat(bucket)
if err != nil && os.IsNotExist(err) {
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
@@ -1035,3 +1043,83 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int)
Prefix: &prefix,
}, nil
}
func (p *Posix) GetTags(bucket, object string) (map[string]string, error) {
_, err := os.Stat(bucket)
if err != nil && errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
return p.getXattrTags(bucket, object)
}
func (p *Posix) getXattrTags(bucket, object string) (map[string]string, error) {
tags := make(map[string]string)
b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if isNoAttr(err) {
return tags, nil
}
if err != nil {
return nil, fmt.Errorf("get tags: %w", err)
}
err = json.Unmarshal(b, &tags)
if err != nil {
return nil, fmt.Errorf("unmarshal tags: %w", err)
}
return tags, nil
}
func (p *Posix) SetTags(bucket, object string, tags map[string]string) error {
_, err := os.Stat(bucket)
if err != nil && errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return fmt.Errorf("stat bucket: %w", err)
}
if tags == nil {
return xattr.Remove(filepath.Join(bucket, object), "user."+tagHdr)
}
b, err := json.Marshal(tags)
if err != nil {
return fmt.Errorf("marshal tags: %w", err)
}
err = xattr.Set(filepath.Join(bucket, object), "user."+tagHdr, b)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return fmt.Errorf("set tags: %w", err)
}
return nil
}
func (p *Posix) RemoveTags(bucket, object string) error {
return p.SetTags(bucket, object, nil)
}
func isNoAttr(err error) bool {
if err == nil {
return false
}
xerr, ok := err.(*xattr.Error)
if ok && xerr.Err == xattr.ENOATTR {
return true
}
if err == syscall.ENODATA {
return true
}
return false
}