Merge pull request #2377 from RaduBerinde/posix-no-chdir

posix: add AbsolutePaths option for embedding without chdir
This commit is contained in:
Ben McClelland
2026-09-10 17:19:45 -07:00
committed by GitHub
12 changed files with 994 additions and 132 deletions
+15
View File
@@ -45,3 +45,18 @@ type MetadataStorer interface {
// directory for an object is renamed so that metadata stays in sync.
RenameObject(bucket, oldObject, newObject string) error
}
// RootDirSetter is implemented by metadata storers that keep metadata on the
// bucket and object files themselves and therefore need to know where those
// files live. A backend that does not change the process working directory
// to its root calls WithRootDir with the absolute path of the root before
// using the storer; the returned storer resolves bucket names against that
// directory instead of the working directory. Storers that keep metadata
// elsewhere (SideCar, NoMeta) do not implement it.
//
// A type that embeds XattrMeta inherits its WithRootDir, which returns a bare
// XattrMeta and so drops the outer type; such a type must provide its own
// WithRootDir.
type RootDirSetter interface {
WithRootDir(rootdir string) MetadataStorer
}
+55 -5
View File
@@ -31,7 +31,41 @@ var (
ErrNoSuchKey = errors.New("no such key")
)
type XattrMeta struct{}
// XattrMeta stores metadata as extended attributes on the bucket and object
// files. The zero value resolves bucket names against the process working
// directory, which the posix backend sets to its root directory; a backend
// that keeps the working directory supplies the root through WithRootDir.
type XattrMeta struct {
// rootdir is the absolute path bucket names are resolved under, or ""
// to resolve them against the process working directory.
rootdir string
}
var _ RootDirSetter = XattrMeta{}
// WithRootDir returns a copy of x that resolves bucket names against rootdir.
func (x XattrMeta) WithRootDir(rootdir string) MetadataStorer {
x.rootdir = rootdir
return x
}
// path returns the filesystem path holding the attributes of object in
// bucket (of the bucket itself when object is empty).
//
// The bucket argument is normally a bucket name, resolved under the root
// directory (or the working directory when no root is set). The versioning
// code instead passes the absolute path of a bucket's versioning directory;
// an absolute bucket is used as given.
func (x XattrMeta) path(bucket, object string) (string, error) {
if filepath.IsAbs(bucket) {
return filepath.Join(bucket, object), nil
}
if bucket == "" || bucket == "." || bucket == ".." {
// Would resolve to the root directory itself or its parent.
return "", fmt.Errorf("xattr metadata: invalid bucket name %q", bucket)
}
return filepath.Join(x.rootdir, bucket, object), nil
}
// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
@@ -43,7 +77,11 @@ func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute strin
return b, err
}
b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
name, err := x.path(bucket, object)
if err != nil {
return nil, err
}
b, err := xattr.Get(name, xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return nil, ErrNoSuchKey
}
@@ -63,7 +101,11 @@ func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string,
return err
}
err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
name, err := x.path(bucket, object)
if err != nil {
return err
}
err = xattr.Set(name, xattrPrefix+attribute, value)
if errors.Is(err, syscall.EROFS) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
@@ -75,7 +117,11 @@ func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string,
// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
name, err := x.path(bucket, object)
if err != nil {
return err
}
err = xattr.Remove(name, xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return ErrNoSuchKey
}
@@ -99,7 +145,11 @@ func (x XattrMeta) RenameObject(_, _, _ string) error {
// ListAttributes lists all attributes for an object in a bucket.
func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
attrs, err := xattr.List(filepath.Join(bucket, object))
name, err := x.path(bucket, object)
if err != nil {
return nil, err
}
attrs, err := xattr.List(name)
if err != nil {
return nil, err
}
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import (
"path/filepath"
"testing"
)
func TestXattrMetaPath(t *testing.T) {
root := t.TempDir()
abs := filepath.Join(t.TempDir(), "versions", "bucket")
tests := []struct {
name string
x XattrMeta
bucket string
object string
want string
wantErr bool
}{
{name: "bucket under root", x: XattrMeta{}.WithRootDir(root).(XattrMeta), bucket: "b", object: "", want: filepath.Join(root, "b")},
{name: "object under root", x: XattrMeta{}.WithRootDir(root).(XattrMeta), bucket: "b", object: "d/o", want: filepath.Join(root, "b", "d", "o")},
{name: "absolute bucket ignores root", x: XattrMeta{}.WithRootDir(root).(XattrMeta), bucket: abs, object: "o", want: filepath.Join(abs, "o")},
{name: "absolute bucket without root", x: XattrMeta{}, bucket: abs, object: "o", want: filepath.Join(abs, "o")},
{name: "relative bucket without root is cwd-relative", x: XattrMeta{}, bucket: "b", object: "o", want: filepath.Join("b", "o")},
{name: "empty bucket", x: XattrMeta{}.WithRootDir(root).(XattrMeta), bucket: "", object: "o", wantErr: true},
{name: "dot bucket", x: XattrMeta{}.WithRootDir(root).(XattrMeta), bucket: ".", object: "", wantErr: true},
{name: "dotdot bucket", x: XattrMeta{}.WithRootDir(root).(XattrMeta), bucket: "..", object: "", wantErr: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, err := tc.x.path(tc.bucket, tc.object)
if tc.wantErr {
if err == nil {
t.Fatalf("path(%q, %q) = %q, want error", tc.bucket, tc.object, got)
}
return
}
if err != nil {
t.Fatalf("path(%q, %q): %v", tc.bucket, tc.object, err)
}
if got != tc.want {
t.Fatalf("path(%q, %q) = %q, want %q", tc.bucket, tc.object, got, tc.want)
}
})
}
}
+4 -2
View File
@@ -28,8 +28,10 @@ import (
func handleParentDirError(name string) error {
dir := filepath.Dir(name)
// Walk up the directory hierarchy
for dir != "." && dir != "/" {
// Walk up the directory hierarchy until Dir returns its argument
// unchanged: "." for a relative path, the volume root for an absolute
// one.
for dir != filepath.Dir(dir) {
d, statErr := os.Stat(dir)
if statErr == nil {
// Path component exists
+204 -105
View File
@@ -57,6 +57,10 @@ type Posix struct {
rootfd *os.File
rootdir string
// pathRoot is the directory bucket paths are built from: rootdir when
// PosixOpts.AbsolutePaths is set, otherwise "" because the process
// working directory is the root directory. See BucketPath.
pathRoot string
// chownuid/gid enable chowning of files to the account uid/gid
// when objects are uploaded
@@ -216,6 +220,15 @@ type PosixOpts struct {
ChownGID bool
// BucketLinks enables symlinks to directories to be treated as buckets
BucketLinks bool
// AbsolutePaths makes the backend address every bucket and object by an
// absolute path under the root directory instead of changing the
// process working directory to the root and using relative paths. Use it
// when embedding the gateway in a process that must keep its working
// directory, such as a Go test that reads test data by relative path.
// Absolute paths cost slightly more per filesystem operation. A relative
// VersioningDir or SideCarDir is then resolved against the working
// directory rather than the root directory.
AbsolutePaths bool
//VersioningDir sets the version directory to enable object versioning
VersioningDir string
// NewDirPerm specifies the permission to set on newly created directories.
@@ -278,7 +291,13 @@ type PosixOpts struct {
DataIntegrityEtag bool
}
func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
// New returns a backend serving buckets from the directories under rootdir.
//
// By default New changes the process working directory to rootdir and
// addresses every bucket and object by a path relative to it. With
// opts.AbsolutePaths the working directory is left alone and paths are built
// from the absolute root directory instead (see BucketPath and ObjectPath).
func New(rootdir string, ms meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
ioBufferSize := ioBufferSizeOrDefault(opts.IOBufferSize)
rootdirAbs, err := filepath.Abs(rootdir)
if err != nil {
@@ -289,14 +308,26 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
return nil, fmt.Errorf("sidecar directory cannot be inside the gateway root directory")
}
err = os.Chdir(rootdirAbs)
if err != nil {
return nil, fmt.Errorf("chdir %v: %w", rootdir, err)
// A storer that keeps metadata on the object files (xattr) resolves
// bucket names the same way this backend does: relative to the working
// directory by default, under the absolute root with AbsolutePaths.
var pathRoot string
if opts.AbsolutePaths {
pathRoot = rootdirAbs
if rs, ok := ms.(meta.RootDirSetter); ok {
ms = rs.WithRootDir(rootdirAbs)
}
}
f, err := os.Open(rootdirAbs)
if err != nil {
return nil, fmt.Errorf("open %v: %w", rootdir, err)
if !opts.AbsolutePaths {
// Change directory before resolving the versioning and sidecar
// directories: a relative one is then relative to the root, which is
// also how a SideCar storer constructed with the same relative path
// will resolve it once the process runs from the root.
err = os.Chdir(rootdirAbs)
if err != nil {
return nil, fmt.Errorf("chdir %v: %w", rootdir, err)
}
}
var versioningdirAbs string
@@ -317,6 +348,23 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
}
}
// Opened last so that no error path below has to close it.
f, err := os.Open(rootdirAbs)
if err != nil {
return nil, fmt.Errorf("open %v: %w", rootdir, err)
}
if opts.AbsolutePaths {
// Without the chdir above nothing else rejects a root that is not
// a directory.
if fi, err := f.Stat(); err != nil || !fi.IsDir() {
f.Close()
if err != nil {
return nil, fmt.Errorf("stat %v: %w", rootdir, err)
}
return nil, fmt.Errorf("%v is not a directory", rootdir)
}
}
if versioningdirAbs != "" {
fmt.Println("Bucket versioning enabled with directory:", versioningdirAbs)
}
@@ -349,9 +397,10 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
}
return &Posix{
meta: meta,
meta: ms,
rootfd: f,
rootdir: rootdirAbs,
pathRoot: pathRoot,
euid: euid,
egid: egid,
chownuid: opts.ChownUID,
@@ -378,6 +427,37 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
}, nil
}
// BucketPath returns the filesystem path of the bucket directory: the bucket
// name itself by default (the process working directory is the root
// directory), or the absolute path under the root with
// PosixOpts.AbsolutePaths.
//
// The versioning code passes the absolute path of a bucket's versioning
// directory where a bucket name is expected, and the metadata storer accepts
// the same substitution; an absolute bucket is therefore returned as given.
func (p *Posix) BucketPath(bucket string) string {
if p.pathRoot == "" || filepath.IsAbs(bucket) {
return bucket
}
return filepath.Join(p.pathRoot, bucket)
}
// rootPath returns the filesystem path of the root directory in the form
// BucketPath builds bucket paths from.
func (p *Posix) rootPath() string {
if p.pathRoot == "" {
return "."
}
return p.pathRoot
}
// ObjectPath returns the filesystem path of object within bucket. object is
// any path relative to the bucket directory: an object key, a multipart
// upload's temporary directory, or a part file within one.
func (p *Posix) ObjectPath(bucket, object string) string {
return filepath.Join(p.BucketPath(bucket), object)
}
// concurrencyOrDefault returns n if it is positive, otherwise defaultConcurrency.
func concurrencyOrDefault(n int) int {
if n > 0 {
@@ -514,7 +594,7 @@ func (p *Posix) validateVersionId(versionId string) error {
}
func (p *Posix) doesBucketAndObjectExist(bucket, object string) error {
_, err := os.Stat(bucket)
_, err := os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -522,7 +602,7 @@ func (p *Posix) doesBucketAndObjectExist(bucket, object string) error {
return fmt.Errorf("stat bucket: %w", err)
}
_, err = os.Stat(filepath.Join(bucket, object))
_, err = os.Stat(p.ObjectPath(bucket, object))
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
@@ -540,7 +620,7 @@ func (p *Posix) ListBuckets(ctx context.Context, input s3response.ListBucketsInp
}
defer release()
fis, err := listBucketFileInfos(p.bucketlinks)
fis, err := listBucketFileInfos(p.rootPath(), p.bucketlinks)
if err != nil {
return s3response.ListAllMyBucketsResult{}, fmt.Errorf("listBucketFileInfos : %w", err)
}
@@ -609,8 +689,22 @@ func (p *Posix) ListBuckets(ctx context.Context, input s3response.ListBucketsInp
}, nil
}
// IsBucketValid reports whether bucket may be used as a bucket name by this
// backend. Backends built on Posix that add their own entry points must check
// bucket names with it before building paths from them.
func (p *Posix) IsBucketValid(bucket string) bool {
return p.isBucketValid(bucket)
}
func (p *Posix) isBucketValid(bucket string) bool {
if bucket == objLockDir {
// Regardless of validation mode, a bucket name must name a single
// directory entry under the root: "", "." and ".." would resolve to the
// root or its parent (see BucketPath), a separator or an absolute path
// would escape it. Callers that need the versioning directory pass it in
// place of the bucket only after this check.
if bucket == "" || bucket == "." || bucket == ".." || bucket == objLockDir ||
filepath.IsAbs(bucket) || strings.ContainsRune(bucket, '/') ||
strings.ContainsRune(bucket, os.PathSeparator) {
return false
}
@@ -630,7 +724,7 @@ func (p *Posix) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.
if !p.isBucketValid(*input.Bucket) {
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, *input.Bucket)
}
_, err = os.Lstat(*input.Bucket)
_, err = os.Lstat(p.BucketPath(*input.Bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, *input.Bucket)
}
@@ -666,7 +760,7 @@ func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, a
return err
}
err = os.Mkdir(bucket, p.newDirPerm)
err = os.Mkdir(p.BucketPath(bucket), p.newDirPerm)
if err != nil && os.IsExist(err) {
aclJSON, err := p.meta.RetrieveAttribute(nil, bucket, "", aclkey)
if errors.Is(err, meta.ErrNoSuchKey) {
@@ -710,7 +804,7 @@ func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, a
}()
if doChown {
err = os.Chown(bucket, uid, gid)
err = os.Chown(p.BucketPath(bucket), uid, gid)
if err != nil {
return p.chownErr(bucket, uid, gid, err)
}
@@ -771,7 +865,7 @@ func (p *Posix) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, a
// are logged rather than returned — the caller is already failing with the
// error that matters, and reporting a cleanup failure instead would hide it.
func (p *Posix) removePartialBucket(bucket string) {
if err := os.RemoveAll(bucket); err != nil {
if err := os.RemoveAll(p.BucketPath(bucket)); err != nil {
debuglogger.Logf("failed to remove partially created bucket (%q): %v", bucket, err)
}
if err := p.meta.DeleteAttributes(bucket, ""); err != nil {
@@ -831,7 +925,7 @@ func (p *Posix) isBucketEmpty(bucket string) error {
}
}
ents, err := os.ReadDir(bucket)
ents, err := os.ReadDir(p.BucketPath(bucket))
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("readdir bucket: %w", err)
}
@@ -864,7 +958,7 @@ func (p *Posix) DeleteBucket(ctx context.Context, bucket string) error {
}
// Remove the bucket
err = os.RemoveAll(bucket)
err = os.RemoveAll(p.BucketPath(bucket))
if err != nil {
return fmt.Errorf("remove bucket: %w", err)
}
@@ -896,7 +990,7 @@ func (p *Posix) PutBucketOwnershipControls(ctx context.Context, bucket string, o
if !p.isBucketValid(bucket) {
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -922,7 +1016,7 @@ func (p *Posix) GetBucketOwnershipControls(ctx context.Context, bucket string) (
if !p.isBucketValid(bucket) {
return ownship, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return ownship, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -950,7 +1044,7 @@ func (p *Posix) DeleteBucketOwnershipControls(ctx context.Context, bucket string
if !p.isBucketValid(bucket) {
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -983,7 +1077,7 @@ func (p *Posix) PutBucketVersioning(ctx context.Context, bucket string, status t
if !p.versioningEnabled() {
return s3err.GetAPIError(s3err.ErrVersioningNotConfigured)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -1035,7 +1129,7 @@ func (p *Posix) GetBucketVersioning(ctx context.Context, bucket string) (s3respo
return s3response.GetBucketVersioningOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.GetBucketVersioningOutput{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -1140,7 +1234,7 @@ func isRemovableAttr(attr string) bool {
// Creates a new copy(version) of an object in the versioning directory
func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Account, removeAttributes bool) (versionPath string, err error) {
sf, err := os.Open(filepath.Join(bucket, key))
sf, err := os.Open(p.ObjectPath(bucket, key))
if err != nil {
return "", err
}
@@ -1244,7 +1338,7 @@ func (p *Posix) ListObjectVersions(ctx context.Context, input *s3.ListObjectVers
max = int(*input.MaxKeys)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.ListVersionsResult{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -1252,7 +1346,7 @@ func (p *Posix) ListObjectVersions(ctx context.Context, input *s3.ListObjectVers
return s3response.ListVersionsResult{}, fmt.Errorf("stat bucket: %w", err)
}
fileSystem := os.DirFS(bucket)
fileSystem := os.DirFS(p.BucketPath(bucket))
results, err := backend.WalkVersions(ctx, fileSystem, prefix, delim, keyMarker, versionIdMarker, max,
p.fileToObjVersions(bucket), []string{MetaTmpDir})
if err != nil {
@@ -1291,7 +1385,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
// data file simply doesn't exist — the two cases are indistinguishable
// from metadata alone. Verify the data file directly so callers
// receive the correct NoSuchVersion / NoSuchKey error.
if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
if _, statErr := os.Stat(p.ObjectPath(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
if versionId != "" {
return s3err.GetAPIError(s3err.ErrNoSuchVersion)
}
@@ -1720,7 +1814,7 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create
return s3response.InitiateMultipartUploadResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.InitiateMultipartUploadResult{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -1747,7 +1841,7 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create
// multiple uploads for same object name allowed,
// they will all go into the same hashed name directory
objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", objNameSum))
tmppath := filepath.Join(bucket, objdir)
tmppath := p.ObjectPath(bucket, objdir)
// the unique upload id is a directory for all of the parts
// associated with this specific multipart upload
err = os.MkdirAll(filepath.Join(tmppath, uploadID), p.newDirPerm)
@@ -2014,7 +2108,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
return res, "", s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err := os.Stat(bucket)
_, err := os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return res, "", s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -2027,7 +2121,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
// the same ETag, so it will either find the directory still present (still
// processing) or gone (already completed) and react accordingly.
sum := sha256.Sum256([]byte(object))
objdirFull := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
objdirFull := filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
uploadIDDir := filepath.Join(objdirFull, uploadID)
// Compute the default multipart ETag token used for claim naming.
// In standard mode this is the S3-compatible multipart MD5 ETag; in
@@ -2087,7 +2181,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
}
partObjPath := filepath.Join(objdir, uploadName, fmt.Sprintf("%v", *part.PartNumber))
fi, err := os.Lstat(filepath.Join(bucket, partObjPath))
fi, err := os.Lstat(p.ObjectPath(bucket, partObjPath))
if err != nil {
return "", s3err.GetInvalidPartErr(uploadID, *part.PartNumber, backend.GetStringFromPtr(part.ETag))
}
@@ -2152,7 +2246,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
}, "", nil
}
// Directory is gone: the concurrent call already completed and cleaned up.
if _, statErr := os.Stat(filepath.Join(bucket, object)); statErr == nil {
if _, statErr := os.Stat(p.ObjectPath(bucket, object)); statErr == nil {
etag := multipartClaimToken
if p.dataIntegrityEtag {
etagBytes, etagErr := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
@@ -2286,7 +2380,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
partNumber = *part.PartNumber
partObjPath := filepath.Join(objdir, activeUploadName, fmt.Sprintf("%v", *part.PartNumber))
fullPartPath := filepath.Join(bucket, partObjPath)
fullPartPath := p.ObjectPath(bucket, partObjPath)
fi, err := os.Lstat(fullPartPath)
if err != nil {
return res, "", s3err.GetInvalidPartErr(uploadID, *part.PartNumber, backend.GetStringFromPtr(part.ETag))
@@ -2442,7 +2536,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
finalEtag = fmt.Sprintf("\"%s-%s\"", strings.ToUpper(string(checksums.Algorithm)), value)
}
f, err := p.openTmpFile(filepath.Join(bucket, MetaTmpDir), bucket, object,
f, err := p.openTmpFile(p.ObjectPath(bucket, MetaTmpDir), bucket, object,
totalsize, acct, skipFalloc, p.forceNoTmpFile, odirectNotAllowed)
if err != nil {
if errors.Is(err, syscall.EDQUOT) {
@@ -2458,7 +2552,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
var abortOnErrSet bool
for _, part := range parts {
partObjPath := filepath.Join(objdir, activeUploadName, fmt.Sprintf("%v", *part.PartNumber))
fullPartPath := filepath.Join(bucket, partObjPath)
fullPartPath := p.ObjectPath(bucket, partObjPath)
pf, err := os.Open(fullPartPath)
if err != nil {
return res, "", fmt.Errorf("open part %v: %v", *part.PartNumber, err)
@@ -2486,11 +2580,11 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
if !abortOnErrSet {
defer func() {
// cleanup tmp dirs
os.RemoveAll(filepath.Join(bucket, objdir, activeUploadName))
os.RemoveAll(filepath.Join(p.BucketPath(bucket), objdir, activeUploadName))
// use Remove for objdir in case there are still other
// uploads for same object name outstanding, this will
// fail if there are any
os.Remove(filepath.Join(bucket, objdir))
os.Remove(p.ObjectPath(bucket, objdir))
}()
}
abortOnErrSet = true
@@ -2539,7 +2633,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
return res, "", err
}
objname := filepath.Join(bucket, object)
objname := p.ObjectPath(bucket, object)
dir := filepath.Dir(objname)
if dir != "" {
uid, gid, doChown := p.getChownIDs(acct)
@@ -2645,10 +2739,10 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
}
// cleanup tmp dirs
os.RemoveAll(filepath.Join(bucket, objdir, activeUploadName))
os.RemoveAll(filepath.Join(p.BucketPath(bucket), objdir, activeUploadName))
// use Remove for objdir in case there are still other uploads
// for same object name outstanding, this will fail if there are any
os.Remove(filepath.Join(bucket, objdir))
os.Remove(p.ObjectPath(bucket, objdir))
return s3response.CompleteMultipartUploadResult{
Bucket: &bucket,
@@ -2770,7 +2864,7 @@ func numberOfChecksums(part types.CompletedPart) (int, string) {
func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) {
sum := sha256.Sum256([]byte(object))
objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
objdir := filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err := os.Stat(filepath.Join(objdir, uploadID))
if errors.Is(err, fs.ErrNotExist) {
@@ -3015,7 +3109,7 @@ func (p *Posix) AbortMultipartUpload(ctx context.Context, mpu *s3.AbortMultipart
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -3024,7 +3118,7 @@ func (p *Posix) AbortMultipartUpload(ctx context.Context, mpu *s3.AbortMultipart
}
sum := sha256.Sum256([]byte(object))
objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
objdir := filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
f, err := os.Stat(filepath.Join(objdir, uploadID))
if err != nil {
@@ -3084,7 +3178,7 @@ func (p *Posix) ListMultipartUploads(ctx context.Context, mpu *s3.ListMultipartU
}
maxUploads := int(*mpu.MaxUploads)
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return lmu, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -3093,7 +3187,7 @@ func (p *Posix) ListMultipartUploads(ctx context.Context, mpu *s3.ListMultipartU
}
// ignore readdir error and use the empty list returned
objs, _ := os.ReadDir(filepath.Join(bucket, MetaTmpMultipartDir))
objs, _ := os.ReadDir(p.ObjectPath(bucket, MetaTmpMultipartDir))
var uploads []s3response.Upload
@@ -3116,7 +3210,7 @@ func (p *Posix) ListMultipartUploads(ctx context.Context, mpu *s3.ListMultipartU
continue
}
upids, err := os.ReadDir(filepath.Join(bucket, MetaTmpMultipartDir, obj.Name()))
upids, err := os.ReadDir(filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, obj.Name()))
if err != nil {
continue
}
@@ -3226,7 +3320,7 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp
}
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return lpr, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -3240,7 +3334,7 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp
}
objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
tmpdir := filepath.Join(bucket, objdir)
tmpdir := p.ObjectPath(bucket, objdir)
ents, err := os.ReadDir(filepath.Join(tmpdir, uploadID))
if errors.Is(err, fs.ErrNotExist) {
@@ -3250,7 +3344,7 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp
return lpr, fmt.Errorf("readdir upload: %w", err)
}
checksum, err := p.retrieveChecksums(nil, tmpdir, uploadID)
checksum, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID))
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return lpr, fmt.Errorf("get mp checksum: %w", err)
}
@@ -3291,7 +3385,7 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp
continue
}
fi, err := os.Lstat(filepath.Join(bucket, partPath))
fi, err := os.Lstat(p.ObjectPath(bucket, partPath))
if err != nil {
continue
}
@@ -3382,7 +3476,7 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
}
r := input.Body
_, err := os.Stat(bucket)
_, err := os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -3394,7 +3488,7 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
mpPath := filepath.Join(objdir, uploadID)
_, err = os.Stat(filepath.Join(bucket, mpPath))
_, err = os.Stat(p.ObjectPath(bucket, mpPath))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetNoSuchUploadErr(uploadID)
}
@@ -3404,7 +3498,7 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
partPath := filepath.Join(mpPath, fmt.Sprintf("%v", *part))
f, err := p.openTmpFile(filepath.Join(bucket, objdir),
f, err := p.openTmpFile(p.ObjectPath(bucket, objdir),
bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile, odirectAllowed)
if err != nil {
if errors.Is(err, syscall.EDQUOT) {
@@ -3706,7 +3800,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyPartResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, *upi.Bucket)
}
_, err = os.Stat(*upi.Bucket)
_, err = os.Stat(p.BucketPath(*upi.Bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, *upi.Bucket)
}
@@ -3717,7 +3811,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
sum := sha256.Sum256([]byte(*upi.Key))
objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err = os.Stat(filepath.Join(*upi.Bucket, objdir, *upi.UploadId))
_, err = os.Stat(filepath.Join(p.BucketPath(*upi.Bucket), objdir, *upi.UploadId))
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetNoSuchUploadErr(*upi.UploadId)
}
@@ -3737,8 +3831,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
if err := p.validateVersionId(srcVersionId); err != nil {
return s3response.CopyPartResult{}, err
}
if !p.isBucketValid(srcBucket) {
return s3response.CopyPartResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, srcBucket)
}
_, err = os.Stat(srcBucket)
_, err = os.Stat(p.BucketPath(srcBucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, srcBucket)
}
@@ -3784,7 +3881,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
}
}
objPath := filepath.Join(srcBucket, srcObject)
objPath := p.ObjectPath(srcBucket, srcObject)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) {
if p.versioningEnabled() && vEnabled {
@@ -3833,7 +3930,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyPartResult{}, err
}
f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),
f, err := p.openTmpFile(p.ObjectPath(*upi.Bucket, objdir),
*upi.Bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile, odirectNotAllowed)
if err != nil {
if errors.Is(err, syscall.EDQUOT) {
@@ -3859,7 +3956,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyPartResult{}, fmt.Errorf("retrieve mp checksums: %w", err)
}
checksums, err := p.retrieveChecksums(nil, objPath, "")
checksums, err := p.retrieveChecksums(nil, srcBucket, srcObject)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return s3response.CopyPartResult{}, fmt.Errorf("retrieve object part checksums: %w", err)
}
@@ -3959,7 +4056,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyPartResult{}, fmt.Errorf("link object in namespace: %w", err)
}
fi, err = os.Stat(filepath.Join(*upi.Bucket, partPath))
fi, err = os.Stat(p.ObjectPath(*upi.Bucket, partPath))
if err != nil {
return s3response.CopyPartResult{}, fmt.Errorf("stat part path: %w", err)
}
@@ -4034,7 +4131,7 @@ func (p *Posix) snapshotObjVersion(bucket, key string, vStatus types.BucketVersi
return nil
}
d, err := os.Stat(filepath.Join(bucket, key))
d, err := os.Stat(p.ObjectPath(bucket, key))
if err != nil || d.IsDir() {
// nothing to snapshot
return nil
@@ -4087,7 +4184,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
if !p.isBucketValid(*po.Bucket) {
return s3response.PutObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, *po.Bucket)
}
_, err := os.Stat(*po.Bucket)
_, err := os.Stat(p.BucketPath(*po.Bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.PutObjectOutput{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, *po.Bucket)
}
@@ -4100,7 +4197,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
return s3response.PutObjectOutput{}, err
}
name := filepath.Join(*po.Bucket, *po.Key)
name := p.ObjectPath(*po.Bucket, *po.Key)
// Fast-fail precondition check before the request body is staged. This
// is only advisory: the authoritative check is repeated while holding
@@ -4293,7 +4390,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
return s3response.PutObjectOutput{}, fmt.Errorf("stat object: %w", err)
}
f, err := p.openTmpFile(filepath.Join(*po.Bucket, MetaTmpDir),
f, err := p.openTmpFile(p.ObjectPath(*po.Bucket, MetaTmpDir),
*po.Bucket, *po.Key, contentLength, acct, doFalloc, p.forceNoTmpFile, odirectAllowed)
if err != nil {
if errors.Is(err, syscall.EDQUOT) {
@@ -4587,7 +4684,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return nil, err
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -4595,7 +4692,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return nil, fmt.Errorf("stat bucket: %w", err)
}
objpath := filepath.Join(bucket, object)
objpath := p.ObjectPath(bucket, object)
vStatus, err := p.getBucketVersioningStatus(ctx, bucket)
if err != nil {
@@ -4605,7 +4702,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
evalPreconditions := func(f os.FileInfo, bucket, object string) error {
var err error
if f == nil {
f, err = os.Stat(filepath.Join(bucket, object))
f, err = os.Stat(p.ObjectPath(bucket, object))
if err != nil {
return nil
}
@@ -4714,7 +4811,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
// Also handle ENOTDIR: when a key such as "foo/bar" is requested
// but "foo" is a regular file (not a directory), the path cannot
// contain any object.
_, statErr := os.Stat(filepath.Join(bucket, object))
_, statErr := os.Stat(p.ObjectPath(bucket, object))
if errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil
}
@@ -4773,7 +4870,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
acct = auth.Account{}
}
f, err := p.openTmpFile(filepath.Join(bucket, MetaTmpDir),
f, err := p.openTmpFile(p.ObjectPath(bucket, MetaTmpDir),
bucket, object, srcObjVersion.Size(), acct, doFalloc,
p.forceNoTmpFile, odirectNotAllowed)
if err != nil {
@@ -4901,7 +4998,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
if isErrDirNotEmpty(err) {
// If the directory object has been uploaded explicitly
// remove the directory object (remove the ETag)
_, err = p.meta.RetrieveAttribute(nil, objpath, "", etagkey)
_, err = p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object etag: %w", err)
}
@@ -4909,7 +5006,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return nil, s3err.GetAPIError(s3err.ErrDirectoryNotEmpty)
}
err = p.meta.DeleteAttribute(objpath, "", etagkey)
err = p.meta.DeleteAttribute(bucket, object, etagkey)
if err != nil {
return nil, fmt.Errorf("delete object etag: %w", err)
}
@@ -4955,7 +5052,7 @@ func (p *Posix) removeParents(bucket, object string) {
break
}
err = os.Remove(filepath.Join(bucket, parent))
err = os.Remove(p.ObjectPath(bucket, parent))
if err != nil {
break
}
@@ -5032,7 +5129,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
if !p.isBucketValid(bucket) {
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -5059,7 +5156,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
}
}
objPath := filepath.Join(bucket, object)
objPath := p.ObjectPath(bucket, object)
fid, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
@@ -5376,7 +5473,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -5403,7 +5500,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
}
}
objPath := filepath.Join(bucket, object)
objPath := p.ObjectPath(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
@@ -5697,7 +5794,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, dstBucket)
}
_, err = os.Stat(srcBucket)
_, err = os.Stat(p.BucketPath(srcBucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, srcBucket)
}
@@ -5744,7 +5841,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
}
}
_, err = os.Stat(dstBucket)
_, err = os.Stat(p.BucketPath(dstBucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, dstBucket)
}
@@ -5752,7 +5849,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
return s3response.CopyObjectOutput{}, fmt.Errorf("stat bucket: %w", err)
}
objPath := joinPathWithTrailer(srcBucket, srcObject)
objPath := joinPathWithTrailer(p.BucketPath(srcBucket), srcObject)
f, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if p.versioningEnabled() && vEnabled {
@@ -5812,7 +5909,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
var xxhash128 *string
var chType types.ChecksumType
dstObjdPath := joinPathWithTrailer(dstBucket, dstObject)
dstObjdPath := joinPathWithTrailer(p.BucketPath(dstBucket), dstObject)
if dstObjdPath == objPath {
if input.MetadataDirective == types.MetadataDirectiveCopy {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
@@ -6085,7 +6182,7 @@ func (p *Posix) ListObjectsParametrized(ctx context.Context, input *s3.ListObjec
return s3response.ListObjectsResult{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err := os.Stat(bucket)
_, err := os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.ListObjectsResult{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6093,7 +6190,7 @@ func (p *Posix) ListObjectsParametrized(ctx context.Context, input *s3.ListObjec
return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err)
}
fileSystem := os.DirFS(bucket)
fileSystem := os.DirFS(p.BucketPath(bucket))
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
customFileToObj(bucket, true), []string{MetaTmpDir})
if err != nil {
@@ -6265,7 +6362,7 @@ func (p *Posix) ListObjectsV2Parametrized(ctx context.Context, input *s3.ListObj
return s3response.ListObjectsV2Result{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err := os.Stat(bucket)
_, err := os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3response.ListObjectsV2Result{}, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6273,7 +6370,7 @@ func (p *Posix) ListObjectsV2Parametrized(ctx context.Context, input *s3.ListObj
return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err)
}
fileSystem := os.DirFS(bucket)
fileSystem := os.DirFS(p.BucketPath(bucket))
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
customFileToObj(bucket, fetchOwner), []string{MetaTmpDir})
if err != nil {
@@ -6307,7 +6404,7 @@ func (p *Posix) PutBucketAcl(ctx context.Context, bucket string, data []byte) er
if !p.isBucketValid(bucket) {
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6333,7 +6430,7 @@ func (p *Posix) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) (
if !p.isBucketValid(*input.Bucket) {
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, *input.Bucket)
}
_, err = os.Stat(*input.Bucket)
_, err = os.Stat(p.BucketPath(*input.Bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, *input.Bucket)
}
@@ -6361,7 +6458,7 @@ func (p *Posix) PutBucketTagging(ctx context.Context, bucket string, tags map[st
if !p.isBucketValid(bucket) {
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6401,7 +6498,7 @@ func (p *Posix) GetBucketTagging(ctx context.Context, bucket string) (map[string
if !p.isBucketValid(bucket) {
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6434,7 +6531,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
if !p.isBucketValid(bucket) {
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6447,7 +6544,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
}
if versionId == "" {
_, err = os.Stat(filepath.Join(bucket, object))
_, err = os.Stat(p.ObjectPath(bucket, object))
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
@@ -6524,7 +6621,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
if !p.isBucketValid(bucket) {
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6537,7 +6634,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
}
if versionId == "" {
_, err = os.Stat(filepath.Join(bucket, object))
_, err = os.Stat(p.ObjectPath(bucket, object))
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
@@ -6626,7 +6723,7 @@ func (p *Posix) PutBucketPolicy(ctx context.Context, bucket string, policy []byt
if !p.isBucketValid(bucket) {
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6665,7 +6762,7 @@ func (p *Posix) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, err
if !p.isBucketValid(bucket) {
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6704,7 +6801,7 @@ func (p *Posix) PutBucketCors(ctx context.Context, bucket string, cors []byte) e
if !p.isBucketValid(bucket) {
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6739,7 +6836,7 @@ func (p *Posix) GetBucketCors(ctx context.Context, bucket string) ([]byte, error
if !p.isBucketValid(bucket) {
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6775,7 +6872,7 @@ func (p *Posix) PutBucketWebsite(ctx context.Context, bucket string, website []b
if !p.isBucketValid(bucket) {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
@@ -6817,7 +6914,7 @@ func (p *Posix) GetBucketWebsite(ctx context.Context, bucket string) ([]byte, er
if !p.isBucketValid(bucket) {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
@@ -6882,7 +6979,7 @@ func (p *Posix) PutObjectLockConfiguration(ctx context.Context, bucket string, c
if !p.isBucketValid(bucket) {
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -6925,7 +7022,7 @@ func (p *Posix) GetObjectLockConfiguration(ctx context.Context, bucket string) (
if !p.isBucketValid(bucket) {
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err = os.Stat(bucket)
_, err = os.Stat(p.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, bucket)
}
@@ -7210,8 +7307,10 @@ func (p *Posix) ChangeBucketOwner(ctx context.Context, bucket, owner string) err
return auth.UpdateBucketACLOwner(ctx, p, bucket, owner)
}
func listBucketFileInfos(bucketlinks bool) ([]fs.FileInfo, error) {
entries, err := os.ReadDir(".")
// listBucketFileInfos returns the file info of every bucket directory under
// rootdir. With bucketlinks, a symlink to a directory counts as a bucket too.
func listBucketFileInfos(rootdir string, bucketlinks bool) ([]fs.FileInfo, error) {
entries, err := os.ReadDir(rootdir)
if err != nil {
return nil, fmt.Errorf("readdir buckets: %w", err)
}
@@ -7228,7 +7327,7 @@ func listBucketFileInfos(bucketlinks bool) ([]fs.FileInfo, error) {
}
if bucketlinks && entry.Type() == fs.ModeSymlink {
fi, err = os.Stat(entry.Name())
fi, err = os.Stat(filepath.Join(rootdir, entry.Name()))
if err != nil {
// skip entries returning errors
continue
@@ -7253,7 +7352,7 @@ func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.
}
defer release()
fis, err := listBucketFileInfos(p.bucketlinks)
fis, err := listBucketFileInfos(p.rootPath(), p.bucketlinks)
if err != nil {
return buckets, fmt.Errorf("listBucketFileInfos: %w", err)
}
+180
View File
@@ -0,0 +1,180 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package posix
import (
"context"
"fmt"
"io"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/s3response"
)
// Metadata-heavy object operations on small objects, where filesystem path
// resolution and metadata lookups dominate rather than data transfer. Run
// with -bench 'Posix' to compare path resolution strategies across branches.
func benchPosix(b *testing.B, mkMeta func(*testing.B) (meta.MetadataStorer, PosixOpts)) (*Posix, context.Context) {
b.Helper()
// New chdirs into the root in default mode; run from a scratch directory
// so the cwd is restored when the benchmark ends.
b.Chdir(b.TempDir())
storer, opts := mkMeta(b)
p, err := New(b.TempDir(), storer, opts)
if err != nil {
b.Fatalf("new posix: %v", err)
}
ctx := context.Background()
bucket := "bucket"
err = p.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: &bucket,
CreateBucketConfiguration: &types.CreateBucketConfiguration{},
}, []byte{})
if err != nil {
b.Fatalf("create bucket: %v", err)
}
return p, ctx
}
// benchMetaModes returns a constructor per metadata storer and path mode:
// "xattr"/"sidecar" use the default chdir-relative paths, "xattr-abs" and
// "sidecar-abs" set PosixOpts.AbsolutePaths.
func benchMetaModes(b *testing.B) map[string]func(*testing.B) (meta.MetadataStorer, PosixOpts) {
modes := map[string]func(*testing.B) (meta.MetadataStorer, PosixOpts){}
for _, abs := range []bool{false, true} {
suffix := ""
if abs {
suffix = "-abs"
}
modes["xattr"+suffix] = func(b *testing.B) (meta.MetadataStorer, PosixOpts) {
return meta.XattrMeta{}, PosixOpts{NewDirPerm: 0755, AbsolutePaths: abs}
}
modes["sidecar"+suffix] = func(b *testing.B) (meta.MetadataStorer, PosixOpts) {
dir := b.TempDir()
sc, err := meta.NewSideCar(dir)
if err != nil {
b.Fatalf("new sidecar: %v", err)
}
return sc, PosixOpts{NewDirPerm: 0755, SideCarDir: dir, AbsolutePaths: abs}
}
}
return modes
}
func benchPut(b *testing.B, p *Posix, ctx context.Context, key, body string) {
b.Helper()
bucket := "bucket"
_, err := p.PutObject(ctx, s3response.PutObjectInput{
Bucket: &bucket,
Key: &key,
Body: strings.NewReader(body),
ContentLength: aws.Int64(int64(len(body))),
})
if err != nil {
b.Fatalf("put %q: %v", key, err)
}
}
func BenchmarkPosixHeadObject(b *testing.B) {
for name, mkMeta := range benchMetaModes(b) {
b.Run(name, func(b *testing.B) {
p, ctx := benchPosix(b, mkMeta)
bucket, key := "bucket", "dir/sub/object"
benchPut(b, p, ctx, key, "hello")
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := p.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &bucket, Key: &key}); err != nil {
b.Fatal(err)
}
}
})
}
}
func BenchmarkPosixGetObject(b *testing.B) {
for name, mkMeta := range benchMetaModes(b) {
b.Run(name, func(b *testing.B) {
p, ctx := benchPosix(b, mkMeta)
bucket, key := "bucket", "dir/sub/object"
benchPut(b, p, ctx, key, "hello")
b.ResetTimer()
for i := 0; i < b.N; i++ {
out, err := p.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &key})
if err != nil {
b.Fatal(err)
}
if _, err := io.Copy(io.Discard, out.Body); err != nil {
b.Fatal(err)
}
out.Body.Close()
}
})
}
}
func BenchmarkPosixPutObject(b *testing.B) {
for name, mkMeta := range benchMetaModes(b) {
b.Run(name, func(b *testing.B) {
p, ctx := benchPosix(b, mkMeta)
b.ResetTimer()
for i := 0; i < b.N; i++ {
benchPut(b, p, ctx, fmt.Sprintf("dir/sub/object-%d", i%64), "hello")
}
})
}
}
func BenchmarkPosixListObjectsV2(b *testing.B) {
for name, mkMeta := range benchMetaModes(b) {
b.Run(name, func(b *testing.B) {
p, ctx := benchPosix(b, mkMeta)
bucket := "bucket"
for i := 0; i < 100; i++ {
benchPut(b, p, ctx, fmt.Sprintf("dir/object-%03d", i), "hello")
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
res, err := p.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket, MaxKeys: aws.Int32(1000), StartAfter: aws.String("")})
if err != nil {
b.Fatal(err)
}
if len(res.Contents) != 100 {
b.Fatalf("listed %d objects", len(res.Contents))
}
}
})
}
}
func BenchmarkPosixHeadBucket(b *testing.B) {
for name, mkMeta := range benchMetaModes(b) {
b.Run(name, func(b *testing.B) {
p, ctx := benchPosix(b, mkMeta)
bucket := "bucket"
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := p.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}); err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -1
View File
@@ -88,7 +88,7 @@ func TestObjectPublishLockHonorsContextWhileWaiting(t *testing.T) {
shard := objLockShard("cancel-wait")
<-p.objLockSlots[shard]
defer func() { p.objLockSlots[shard] <- struct{}{} }()
if _, err := os.Stat(filepath.Join(bucket, objLockDir)); !errors.Is(err, fs.ErrNotExist) {
if _, err := os.Stat(p.ObjectPath(bucket, objLockDir)); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("bucket contains publish lock directory: %v", err)
}
+442
View File
@@ -0,0 +1,442 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package posix
import (
"context"
"encoding/base64"
"encoding/binary"
"errors"
"hash/crc32"
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
// TestDefaultModeChangesWorkingDirectory documents the default behavior: New
// changes the process working directory to the root and addresses buckets by
// relative paths.
func TestDefaultModeChangesWorkingDirectory(t *testing.T) {
work := t.TempDir()
t.Chdir(work)
root := t.TempDir()
p, err := New(root, meta.XattrMeta{}, PosixOpts{})
if err != nil {
t.Fatalf("new posix: %v", err)
}
// Compare by identity: on macOS the temp dir is reached through a
// symlink, so Getwd returns a different spelling of the same directory.
wdInfo, err := os.Stat(".")
if err != nil {
t.Fatalf("stat working directory: %v", err)
}
rootInfo, err := os.Stat(root)
if err != nil {
t.Fatalf("stat root: %v", err)
}
if !os.SameFile(wdInfo, rootInfo) {
wd, _ := os.Getwd()
t.Fatalf("working directory = %q; want root %q", wd, root)
}
if got := p.BucketPath("b"); got != "b" {
t.Fatalf("BucketPath = %q, want %q", got, "b")
}
if got := p.ObjectPath("b", "d/o"); got != filepath.Join("b", "d", "o") {
t.Fatalf("ObjectPath = %q, want %q", got, filepath.Join("b", "d", "o"))
}
}
// TestRootDirIndependentOfWorkingDirectory checks that with AbsolutePaths the
// backend neither changes the process working directory nor depends on it:
// every operation resolves buckets and objects under the root directory it
// was given, so the gateway can be embedded in a process whose working
// directory is elsewhere.
func TestRootDirIndependentOfWorkingDirectory(t *testing.T) {
for name, mkMeta := range metaModes(t) {
t.Run(name, func(t *testing.T) {
// Work from a directory that is neither the gateway root nor
// anything under it, and hand New a relative path to the root so
// that its resolution against the working directory is exercised.
work := t.TempDir()
t.Chdir(work)
root := t.TempDir()
relRoot, err := filepath.Rel(work, root)
if err != nil {
t.Fatalf("relative root: %v", err)
}
storer, opts := mkMeta(t)
opts.AbsolutePaths = true
opts.CopyObjectThreshold = 1 << 20
p, err := New(relRoot, storer, opts)
if err != nil {
t.Fatalf("new posix: %v", err)
}
if wd, err := os.Getwd(); err != nil || wd != work {
t.Fatalf("New changed the working directory to %q (want %q, err %v)", wd, work, err)
}
if got := p.BucketPath("b"); got != filepath.Join(root, "b") {
t.Fatalf("BucketPath = %q, want %q", got, filepath.Join(root, "b"))
}
ctx := context.Background()
bucket, object, body := "bucket", "dir/object", "hello"
createTestBucket(t, p, bucket)
_, err = p.PutObject(ctx, s3response.PutObjectInput{
Bucket: &bucket,
Key: &object,
Body: strings.NewReader(body),
ContentLength: aws.Int64(int64(len(body))),
})
if err != nil {
t.Fatalf("put object: %v", err)
}
// Everything landed under the root, nothing under the working
// directory.
if _, err := os.Stat(filepath.Join(root, bucket, object)); err != nil {
t.Fatalf("object not under root: %v", err)
}
if _, err := os.Stat(filepath.Join(work, bucket)); err == nil {
t.Fatalf("bucket directory created under the working directory")
}
out, err := p.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &object})
if err != nil {
t.Fatalf("get object: %v", err)
}
got, err := io.ReadAll(out.Body)
out.Body.Close()
if err != nil || string(got) != body {
t.Fatalf("get object body = %q, %v; want %q", got, err, body)
}
list, err := p.ListObjects(ctx, &s3.ListObjectsInput{Bucket: &bucket, MaxKeys: aws.Int32(1000)})
if err != nil {
t.Fatalf("list objects: %v", err)
}
if len(list.Contents) != 1 || *list.Contents[0].Key != object {
t.Fatalf("list objects = %+v, want just %q", list.Contents, object)
}
buckets, err := p.ListBuckets(ctx, s3response.ListBucketsInput{IsAdmin: true, MaxBuckets: 1000})
if err != nil {
t.Fatalf("list buckets: %v", err)
}
if len(buckets.Buckets.Bucket) != 1 || buckets.Buckets.Bucket[0].Name != bucket {
t.Fatalf("list buckets = %+v, want just %q", buckets.Buckets.Bucket, bucket)
}
// A multipart upload stores its checksum algorithm in metadata
// keyed by the upload's temporary directory; ListParts must find
// it under the root, and the parts must land there too.
mpKey := "mp/object"
mp, err := p.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &mpKey,
ChecksumAlgorithm: types.ChecksumAlgorithmCrc32,
ChecksumType: types.ChecksumTypeComposite,
})
if err != nil {
t.Fatalf("create multipart upload: %v", err)
}
uploadID := mp.UploadId
crc := make([]byte, 4)
binary.BigEndian.PutUint32(crc, crc32.ChecksumIEEE([]byte(body)))
partCRC := base64.StdEncoding.EncodeToString(crc)
part, err := p.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &mpKey,
UploadId: &uploadID,
PartNumber: aws.Int32(1),
ContentLength: aws.Int64(int64(len(body))),
Body: strings.NewReader(body),
ChecksumCRC32: &partCRC,
})
if err != nil {
t.Fatalf("upload part: %v", err)
}
// The copy source's bucket goes through the same validation as
// every other bucket name.
for _, srcBucket := range []string{"..", "."} {
_, err := p.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &mpKey,
UploadId: &uploadID,
PartNumber: aws.Int32(2),
CopySource: aws.String(srcBucket + "/" + object),
})
if !errors.Is(err, s3err.GetBucketErr(s3err.ErrInvalidBucketName, srcBucket)) {
t.Fatalf("upload part copy from bucket %q: got %v, want InvalidBucketName", srcBucket, err)
}
}
// A valid copy source resolves under the root like any object.
_, err = p.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &mpKey,
UploadId: &uploadID,
PartNumber: aws.Int32(2),
CopySource: aws.String(bucket + "/" + object),
CopySourceRange: aws.String(""),
})
if err != nil {
t.Fatalf("upload part copy: %v", err)
}
if _, err := os.Stat(filepath.Join(root, bucket, MetaTmpMultipartDir)); err != nil {
t.Fatalf("multipart directory not under root: %v", err)
}
lp, err := p.ListParts(ctx, &s3.ListPartsInput{Bucket: &bucket, Key: &mpKey, UploadId: &uploadID, MaxParts: aws.Int32(1000)})
if err != nil {
t.Fatalf("list parts: %v", err)
}
if lp.ChecksumAlgorithm != types.ChecksumAlgorithmCrc32 || len(lp.Parts) != 2 {
t.Fatalf("list parts = algorithm %q, %d parts; want CRC32, 2 parts", lp.ChecksumAlgorithm, len(lp.Parts))
}
_, _, err = p.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &mpKey,
UploadId: &uploadID,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: []types.CompletedPart{{ETag: part.ETag, PartNumber: aws.Int32(1), ChecksumCRC32: &partCRC}},
},
})
if err != nil {
t.Fatalf("complete multipart upload: %v", err)
}
if _, err := os.Stat(filepath.Join(root, bucket, mpKey)); err != nil {
t.Fatalf("multipart object not under root: %v", err)
}
// CopyObject resolves both source and destination under the root.
copyKey := "copy/object"
_, err = p.CopyObject(ctx, s3response.CopyObjectInput{
Bucket: &bucket,
Key: &copyKey,
CopySource: aws.String(bucket + "/" + object),
ExpectedBucketOwner: aws.String(""),
})
if err != nil {
t.Fatalf("copy object: %v", err)
}
if _, err := os.Stat(filepath.Join(root, bucket, copyKey)); err != nil {
t.Fatalf("copied object not under root: %v", err)
}
if _, err := os.Stat(filepath.Join(work, bucket)); err == nil {
t.Fatalf("bucket directory created under the working directory")
}
_, err = p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &copyKey})
if err != nil {
t.Fatalf("delete copied object: %v", err)
}
// Deleting an explicitly created directory object that still has
// children removes only its metadata; the lookup is by bucket and
// object name.
dirKey := "dir/"
_, err = p.PutObject(ctx, s3response.PutObjectInput{
Bucket: &bucket,
Key: &dirKey,
Body: strings.NewReader(""),
ContentLength: aws.Int64(0),
})
if err != nil {
t.Fatalf("put directory object: %v", err)
}
if _, err := p.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &bucket, Key: &dirKey}); err != nil {
t.Fatalf("head directory object: %v", err)
}
_, err = p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &dirKey})
if err != nil {
t.Fatalf("delete non-empty directory object: %v", err)
}
if _, err := os.Stat(filepath.Join(root, bucket, object)); err != nil {
t.Fatalf("child object removed with directory object: %v", err)
}
_, err = p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &dirKey})
if !errors.Is(err, s3err.GetAPIError(s3err.ErrDirectoryNotEmpty)) {
t.Fatalf("second delete of directory object: got %v, want DirectoryNotEmpty", err)
}
_, err = p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &object})
if err != nil {
t.Fatalf("delete object: %v", err)
}
if _, err := os.Stat(filepath.Join(root, bucket, "dir")); !os.IsNotExist(err) {
t.Fatalf("empty parent directory not removed: %v", err)
}
_, err = p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &mpKey})
if err != nil {
t.Fatalf("delete multipart object: %v", err)
}
// Bucket names that would resolve to the root directory, its
// parent, or outside the root are rejected in every validation
// mode (the test backend has posix-level validation off).
for _, name := range []string{"", ".", "..", "a/b", root} {
want := s3err.GetBucketErr(s3err.ErrInvalidBucketName, name)
_, err = p.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &name})
if !errors.Is(err, want) {
t.Fatalf("head bucket %q: got %v, want InvalidBucketName", name, err)
}
if err := p.DeleteBucket(ctx, name); !errors.Is(err, want) {
t.Fatalf("delete bucket %q: got %v, want InvalidBucketName", name, err)
}
}
if _, err := os.Stat(root); err != nil {
t.Fatalf("root directory gone: %v", err)
}
err = p.DeleteBucket(ctx, bucket)
if err != nil {
t.Fatalf("delete bucket: %v", err)
}
if _, err := os.Stat(filepath.Join(root, bucket)); !os.IsNotExist(err) {
t.Fatalf("bucket directory not removed: %v", err)
}
if wd, err := os.Getwd(); err != nil || wd != work {
t.Fatalf("working directory changed to %q (want %q, err %v)", wd, work, err)
}
})
}
}
// TestVersioningDirIndependentOfWorkingDirectory checks the versioning
// directory substitution with AbsolutePaths: the versioning code passes the
// absolute path of a bucket's versioning directory where a bucket name is
// expected, and both the backend and the metadata storer must use it as
// given rather than resolving it under the root or the working directory.
func TestVersioningDirIndependentOfWorkingDirectory(t *testing.T) {
for name, mkMeta := range metaModes(t) {
t.Run(name, func(t *testing.T) {
work := t.TempDir()
t.Chdir(work)
root := t.TempDir()
vdir := t.TempDir()
relRoot, err := filepath.Rel(work, root)
if err != nil {
t.Fatalf("relative root: %v", err)
}
relVdir, err := filepath.Rel(work, vdir)
if err != nil {
t.Fatalf("relative versioning dir: %v", err)
}
storer, opts := mkMeta(t)
opts.VersioningDir = relVdir
opts.AbsolutePaths = true
p, err := New(relRoot, storer, opts)
if err != nil {
t.Fatalf("new posix: %v", err)
}
ctx := context.Background()
bucket, object := "bucket", "object"
createTestBucket(t, p, bucket)
if err := p.PutBucketVersioning(ctx, bucket, types.BucketVersioningStatusEnabled); err != nil {
t.Fatalf("put bucket versioning: %v", err)
}
put := func(body string) string {
t.Helper()
out, err := p.PutObject(ctx, s3response.PutObjectInput{
Bucket: &bucket,
Key: &object,
Body: strings.NewReader(body),
ContentLength: aws.Int64(int64(len(body))),
})
if err != nil {
t.Fatalf("put object %q: %v", body, err)
}
return out.VersionID
}
v1 := put("one")
v2 := put("two")
if v1 == "" || v2 == "" || v1 == v2 {
t.Fatalf("version ids = %q, %q; want two distinct non-empty ids", v1, v2)
}
// The first version was copied into the versioning directory;
// the root holds only the bucket (and the object lock directory)
// and the working directory stays empty.
if _, err := os.Stat(filepath.Join(vdir, bucket)); err != nil {
t.Fatalf("bucket versioning directory missing: %v", err)
}
ents, err := os.ReadDir(root)
if err != nil {
t.Fatalf("read root directory: %v", err)
}
for _, e := range ents {
if e.Name() != bucket && e.Name() != objLockDir {
t.Fatalf("unexpected entry %q in root directory", e.Name())
}
}
if ents, err := os.ReadDir(work); err != nil || len(ents) != 0 {
t.Fatalf("working directory entries = %v, %v; want none", ents, err)
}
get := func(versionID string) string {
t.Helper()
in := &s3.GetObjectInput{Bucket: &bucket, Key: &object}
if versionID != "" {
in.VersionId = &versionID
}
out, err := p.GetObject(ctx, in)
if err != nil {
t.Fatalf("get object version %q: %v", versionID, err)
}
defer out.Body.Close()
got, err := io.ReadAll(out.Body)
if err != nil {
t.Fatalf("read object version %q: %v", versionID, err)
}
return string(got)
}
if got := get(""); got != "two" {
t.Fatalf("current version = %q, want %q", got, "two")
}
if got := get(v1); got != "one" {
t.Fatalf("version %q = %q, want %q", v1, got, "one")
}
// Deleting a specific version removes it from the versioning
// directory.
_, err = p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &object, VersionId: &v1})
if err != nil {
t.Fatalf("delete version: %v", err)
}
_, err = p.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &object, VersionId: &v1})
if !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchVersion)) {
t.Fatalf("get deleted version: got %v, want NoSuchVersion", err)
}
if wd, err := os.Getwd(); err != nil || wd != work {
t.Fatalf("working directory changed to %q (want %q, err %v)", wd, work, err)
}
})
}
}
+2 -2
View File
@@ -71,7 +71,7 @@ func TestVersioningUnconfigured(t *testing.T) {
t.Run("get bucket versioning returns empty config", func(t *testing.T) {
p := newUnversionedGateway(t)
err := os.Mkdir("bucket", 0o755)
err := os.Mkdir(p.BucketPath("bucket"), 0o755)
assert.NoError(t, err)
res, err := p.GetBucketVersioning(context.Background(), "bucket")
@@ -91,7 +91,7 @@ func TestVersioningUnconfigured(t *testing.T) {
t.Run("put bucket versioning not configured", func(t *testing.T) {
p := newUnversionedGateway(t)
err := os.Mkdir("bucket", 0o755)
err := os.Mkdir(p.BucketPath("bucket"), 0o755)
assert.NoError(t, err)
err = p.PutBucketVersioning(context.Background(), "bucket", types.BucketVersioningStatusEnabled)
+7
View File
@@ -52,9 +52,16 @@ type tmpfile struct {
newFilePerm fs.FileMode
}
// openTmpFile opens a temporary file in dir (a filesystem path) that link()
// later publishes as obj within bucket. bucket is a bucket name or, for the
// versioning code, the absolute path of a bucket's versioning directory.
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool, forceNoTmpFile bool, allowODirect odirectPolicy) (*tmpfile, error) {
uid, gid, doChown := p.getChownIDs(acct)
// The tmpfile keeps the bucket directory's path so that link() and its
// fallbacks address it the same way as every other bucket path.
bucket = p.BucketPath(bucket)
if forceNoTmpFile {
return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown, allowODirect)
}
+7
View File
@@ -51,9 +51,16 @@ type tmpfile struct {
doChown bool
}
// openTmpFile opens a temporary file in dir (a filesystem path) that link()
// later publishes as obj within bucket. bucket is a bucket name or, for the
// versioning code, the absolute path of a bucket's versioning directory.
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool, _ bool, allowODirect odirectPolicy) (*tmpfile, error) {
uid, gid, doChown := p.getChownIDs(acct)
// The tmpfile keeps the bucket directory's path so that link() addresses
// it the same way as every other bucket path.
bucket = p.BucketPath(bucket)
if p.enableODirect && bool(allowODirect) {
warnODirectUnsupportedOnce("openTmpFile-nonlinux", os.ErrInvalid)
}
+17 -17
View File
@@ -62,10 +62,6 @@ type ScoutFS struct {
// copies of temporary multipart parts.
disableNoArchive bool
// enable posix level bucket name validations, not needed if the
// frontend handlers are already validating bucket names
validateBucketName bool
// projectIDEnabled enables setting projectid of new buckets and objects
// to the account project id when non-0
projectIDEnabled bool
@@ -93,12 +89,20 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
posixOpts.SetNewFilePerm(opts.NewFilePerm)
}
// Resolve the root before posix.New, which by default changes the
// working directory to it and would make a relative rootdir resolve to
// rootdir/rootdir below.
rootdirAbs, err := filepath.Abs(rootdir)
if err != nil {
return nil, fmt.Errorf("get absolute path of %v: %w", rootdir, err)
}
p, err := posix.New(rootdir, metastore, posixOpts)
if err != nil {
return nil, err
}
f, err := os.Open(rootdir)
f, err := os.Open(rootdirAbs)
if err != nil {
return nil, fmt.Errorf("open %v: %w", rootdir, err)
}
@@ -177,7 +181,7 @@ func (s *ScoutFS) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
return nil
}
f, err := os.Open(*input.Bucket)
f, err := os.Open(s.BucketPath(*input.Bucket))
if err != nil {
debuglogger.InternalError(fmt.Errorf("create bucket %q set project id - open: %v",
*input.Bucket, err))
@@ -202,7 +206,7 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
}
if s.glaciermode {
objPath := filepath.Join(*input.Bucket, *input.Key)
objPath := s.ObjectPath(*input.Bucket, *input.Key)
stclass := types.StorageClassStandard
requestOngoing := stageComplete
@@ -332,11 +336,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
}
func (s *ScoutFS) isBucketValid(bucket string) bool {
if !s.validateBucketName {
return true
}
return backend.IsValidDirectoryName(bucket)
return s.Posix.IsBucketValid(bucket)
}
func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
@@ -347,7 +347,7 @@ func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err := os.Stat(bucket)
_, err := os.Stat(s.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetBucketErr(s3err.ErrNoSuchBucket, *input.Bucket)
}
@@ -355,7 +355,7 @@ func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.
return nil, fmt.Errorf("stat bucket: %w", err)
}
objPath := filepath.Join(bucket, object)
objPath := s.ObjectPath(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
@@ -415,7 +415,7 @@ func (s *ScoutFS) glacierFileToObj(bucket string, fetchOwner bool) backend.GetOb
if err != nil || d.IsDir() {
return res, err
}
objPath := filepath.Join(bucket, path)
objPath := s.ObjectPath(bucket, path)
// Check if there are any offline exents associated with this file.
// If so, we will return the Glacier storage class
st, err := scoutfs.StatMore(objPath)
@@ -442,7 +442,7 @@ func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput)
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
}
_, err := os.Stat(bucket)
_, err := os.Stat(s.BucketPath(bucket))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetBucketErr(s3err.ErrNoSuchBucket, *input.Bucket)
}
@@ -450,7 +450,7 @@ func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput)
return fmt.Errorf("stat bucket: %w", err)
}
err = setStaging(filepath.Join(bucket, object))
err = setStaging(s.ObjectPath(bucket, object))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}